From ca57ee496a45030cbdb962b2b6664ceeeba98dea Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 25 Sep 2026 13:09:38 -0400 Subject: [PATCH 01/58] feat(gax): link T3 attempt spans to parent context and clean up on operation completion --- .../tracing/OpenTelemetryTracingTracer.java | 114 ++++++++++++------ ...OpenTelemetryTracingTracerFactoryTest.java | 1 + .../OpenTelemetryTracingTracerTest.java | 68 ++++++++++- 3 files changed, 143 insertions(+), 40 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index cb41da9ccbeb..dd6e5144c918 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -52,7 +52,10 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final Map attemptAttributes; private final String attemptSpanName; private final ApiTracerContext apiTracerContext; - private @Nullable Span attemptSpan; + private final io.opentelemetry.context.Context parentContext; + private final java.util.concurrent.locks.ReentrantLock lock = + new java.util.concurrent.locks.ReentrantLock(); + private volatile @Nullable Span attemptSpan; @Override public void injectTraceContext(java.util.Map carrier) { @@ -82,6 +85,7 @@ public void injectTraceContext(java.util.Map carrier) { this.apiTracerContext = apiTracerContext; this.attemptSpanName = resolveAttemptSpanName(apiTracerContext); this.attemptAttributes = new HashMap<>(); + this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); } @@ -100,6 +104,7 @@ public void injectTraceContext(java.util.Map carrier) { this.attemptSpanName = attemptSpanName; this.apiTracerContext = apiTracerContext; this.attemptAttributes = new HashMap<>(); + this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); } @@ -124,28 +129,59 @@ private void buildAttributes() { @Override public void attemptStarted(Object request, int attemptNumber) { - Map currentAttemptAttributes = new HashMap<>(this.attemptAttributes); - - if (attemptNumber > 0) { - ApiTracerContext.Transport transport = apiTracerContext.transport(); - if (transport == ApiTracerContext.Transport.GRPC) { - currentAttemptAttributes.put( - ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); - } else if (transport == ApiTracerContext.Transport.HTTP) { - currentAttemptAttributes.put( - ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + Span oldSpan = null; + lock.lock(); + try { + if (attemptSpan != null) { + oldSpan = attemptSpan; + attemptSpan = null; + } + Map currentAttemptAttributes = new HashMap<>(this.attemptAttributes); + + if (attemptNumber > 0) { + ApiTracerContext.Transport transport = apiTracerContext.transport(); + if (transport == ApiTracerContext.Transport.GRPC) { + currentAttemptAttributes.put( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + } else if (transport == ApiTracerContext.Transport.HTTP) { + currentAttemptAttributes.put( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + } } - } - SpanBuilder spanBuilder = tracer.spanBuilder(attemptSpanName); + SpanBuilder spanBuilder = tracer.spanBuilder(attemptSpanName); - // Attempt spans are of the CLIENT kind - spanBuilder.setSpanKind(SpanKind.CLIENT); + // Attempt spans are of the CLIENT kind + spanBuilder.setSpanKind(SpanKind.CLIENT); - // Pass the combined attributes to the new SpanBuilder method - spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); + // Link attempt span to parent context + spanBuilder.setParent(parentContext); + + // Pass the combined attributes to the new SpanBuilder method + spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); + + this.attemptSpan = spanBuilder.startSpan(); + } finally { + lock.unlock(); + } + if (oldSpan != null) { + endAttemptSpan(oldSpan, null); + } + } + + @Override + public void operationSucceeded() { + recordErrorAndEndAttempt(null); + } + + @Override + public void operationCancelled() { + recordErrorAndEndAttempt(new CancellationException()); + } - this.attemptSpan = spanBuilder.startSpan(); + @Override + public void operationFailed(Throwable error) { + recordErrorAndEndAttempt(error); } @Override @@ -155,12 +191,13 @@ public void attemptSucceeded() { @Override public void responseHeadersReceived(java.util.Map headers) { - if (attemptSpan == null) { + Span currentSpan = attemptSpan; + if (currentSpan == null) { return; } long contentLength = extractContentLength(headers); if (contentLength >= 0) { - attemptSpan.setAttribute(ObservabilityAttributes.HTTP_RESPONSE_BODY_SIZE, contentLength); + currentSpan.setAttribute(ObservabilityAttributes.HTTP_RESPONSE_BODY_SIZE, contentLength); } } @@ -216,41 +253,46 @@ public void attemptPermanentFailure(Throwable error) { } private void recordErrorAndEndAttempt(@Nullable Throwable error) { - if (attemptSpan == null) { - return; + Span localAttemptSpan; + lock.lock(); + try { + localAttemptSpan = attemptSpan; + if (localAttemptSpan == null) { + return; + } + attemptSpan = null; + } finally { + lock.unlock(); } + + endAttemptSpan(localAttemptSpan, error); + } + + private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { Map responseAttributes = ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); if (!responseAttributes.isEmpty()) { - attemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); + localAttemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); } if (error != null && !Strings.isNullOrEmpty(error.getMessage())) { - attemptSpan.setAttribute( + localAttemptSpan.setAttribute( ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); } - endAttempt(); - } - - private void endAttempt() { - if (attemptSpan == null) { - return; - } - - attemptSpan.end(); - attemptSpan = null; + localAttemptSpan.end(); } @Override public void requestUrlResolved(String url) { - if (attemptSpan == null) { + Span currentSpan = attemptSpan; + if (currentSpan == null) { return; } String sanitizedUrlString = ObservabilityUtils.sanitizeUrlFull(url); if (sanitizedUrlString.isEmpty()) { return; } - attemptSpan.setAttribute(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, sanitizedUrlString); + currentSpan.setAttribute(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, sanitizedUrlString); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java index 3c78cef6dbd7..de7941522587 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java @@ -75,6 +75,7 @@ void setUp() { when(openTelemetry.getTracer(anyString())).thenReturn(tracer); when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); when(spanBuilder.setSpanKind(any())).thenReturn(spanBuilder); + when(spanBuilder.setParent(any())).thenReturn(spanBuilder); when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); when(spanBuilder.startSpan()).thenReturn(span); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 33fa2efcc0da..41c7627cebe6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -33,6 +33,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -69,10 +71,11 @@ class OpenTelemetryTracingTracerTest { @BeforeEach void setUp() { - when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); - when(spanBuilder.setSpanKind(any(SpanKind.class))).thenReturn(spanBuilder); - when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); - when(spanBuilder.startSpan()).thenReturn(span); + lenient().when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); + lenient().when(spanBuilder.setSpanKind(any(SpanKind.class))).thenReturn(spanBuilder); + lenient().when(spanBuilder.setParent(any())).thenReturn(spanBuilder); + lenient().when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); + lenient().when(spanBuilder.startSpan()).thenReturn(span); openTelemetryTracingTracer = new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); } @@ -680,4 +683,61 @@ void testInjectTraceContext_addsHeaders() { assertThat(carrier.get("traceparent")).contains("00000000000000000000000000000001"); assertThat(carrier.get("traceparent")).contains("0000000000000002"); } + + @Test + void testAttemptStarted_setsParentToParentContext() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(spanBuilder).setParent(any(io.opentelemetry.context.Context.class)); + } + + @Test + void testOperationSucceeded_endsActiveAttemptSpan() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + openTelemetryTracingTracer.operationSucceeded(); + + verify(span).end(); + } + + @Test + void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + openTelemetryTracingTracer.operationFailed(new RuntimeException("operation failed")); + + verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); + verify(span).end(); + } + + @Test + void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + openTelemetryTracingTracer.operationCancelled(); + + ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); + verify(span).setAllAttributes(attrsCaptor.capture()); + verify(span).end(); + + assertThat(attrsCaptor.getValue().asMap()) + .containsEntry( + AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE), + "CANCELLED"); + } + + @Test + void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { + Span span1 = mock(Span.class); + Span span2 = mock(Span.class); + + when(spanBuilder.startSpan()).thenReturn(span1, span2); + + openTelemetryTracingTracer.attemptStarted(new Object(), 0); + + // Start a second attempt before the first attempt was ended + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(span1).end(); + verify(span2, never()).end(); + + // Now complete the second attempt + openTelemetryTracingTracer.attemptSucceeded(); + verify(span2).end(); + } } From d5997216d2a979ae9117a6449fd130e362542b9c Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 25 Sep 2026 13:15:34 -0400 Subject: [PATCH 02/58] fix(gax): prevent new attempt spans after operation completion --- .../gax/tracing/OpenTelemetryTracingTracer.java | 17 ++++++++++++++--- .../tracing/OpenTelemetryTracingTracerTest.java | 9 +++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index dd6e5144c918..3d4f9322da1c 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -55,6 +55,7 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final io.opentelemetry.context.Context parentContext; private final java.util.concurrent.locks.ReentrantLock lock = new java.util.concurrent.locks.ReentrantLock(); + private boolean operationCompleted; private volatile @Nullable Span attemptSpan; @Override @@ -132,6 +133,9 @@ public void attemptStarted(Object request, int attemptNumber) { Span oldSpan = null; lock.lock(); try { + if (operationCompleted) { + return; + } if (attemptSpan != null) { oldSpan = attemptSpan; attemptSpan = null; @@ -171,17 +175,17 @@ public void attemptStarted(Object request, int attemptNumber) { @Override public void operationSucceeded() { - recordErrorAndEndAttempt(null); + recordErrorAndEndAttempt(null, true); } @Override public void operationCancelled() { - recordErrorAndEndAttempt(new CancellationException()); + recordErrorAndEndAttempt(new CancellationException(), true); } @Override public void operationFailed(Throwable error) { - recordErrorAndEndAttempt(error); + recordErrorAndEndAttempt(error, true); } @Override @@ -253,9 +257,16 @@ public void attemptPermanentFailure(Throwable error) { } private void recordErrorAndEndAttempt(@Nullable Throwable error) { + recordErrorAndEndAttempt(error, false); + } + + private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) { Span localAttemptSpan; lock.lock(); try { + if (isOperationComplete) { + operationCompleted = true; + } localAttemptSpan = attemptSpan; if (localAttemptSpan == null) { return; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 41c7627cebe6..2bd2dd320ce4 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -740,4 +740,13 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { openTelemetryTracingTracer.attemptSucceeded(); verify(span2).end(); } + + @Test + void testAttemptStarted_afterOperationCompleted_doesNotStartNewSpan() { + openTelemetryTracingTracer.operationSucceeded(); + + // Attempting to start a new attempt after operation completion should be a no-op + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(spanBuilder, never()).startSpan(); + } } From cda9ac6a292c7a60c15274c225b67aa18a3c0e1d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 25 Sep 2026 13:19:22 -0400 Subject: [PATCH 03/58] fix(gax): use imported ReentrantLock --- .../google/api/gax/tracing/OpenTelemetryTracingTracer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 3d4f9322da1c..03a0b8c2a67d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -39,6 +39,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CancellationException; +import java.util.concurrent.locks.ReentrantLock; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -53,8 +54,7 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final String attemptSpanName; private final ApiTracerContext apiTracerContext; private final io.opentelemetry.context.Context parentContext; - private final java.util.concurrent.locks.ReentrantLock lock = - new java.util.concurrent.locks.ReentrantLock(); + private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; private volatile @Nullable Span attemptSpan; From 18a5cfff2fad42c38c91aad13f281ce0d1908f5b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 13:10:50 -0400 Subject: [PATCH 04/58] chore(gax): add integration test for OpenTelemetry attempt span tracing --- ...TelemetryTracingTracerIntegrationTest.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java new file mode 100644 index 000000000000..31832fbc3d71 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.tracing; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.rpc.LibraryMetadata; +import com.google.api.gax.tracing.ApiTracerContext.Transport; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class OpenTelemetryTracingTracerIntegrationTest { + + private static final String FULL_METHOD_NAME = "google.fake.v1.FakeService/FakeMethod"; + private static final LibraryMetadata LIBRARY_METADATA = + LibraryMetadata.newBuilder() + .setRepository("googleapis/google-cloud-java") + .setArtifactName("google-cloud-fake") + .setVersion("1.0.0") + .build(); + private static final ApiTracerContext TRACER_CONTEXT = + ApiTracerContext.newBuilder() + .setFullMethodName(FULL_METHOD_NAME) + .setTransport(Transport.GRPC) + .setLibraryMetadata(LIBRARY_METADATA) + .setOperationType(OperationType.Unary) + .build(); + + private InMemorySpanExporter spanExporter; + private SdkTracerProvider tracerProvider; + private OpenTelemetrySdk openTelemetrySdk; + private Tracer tracer; + private ApiTracerFactory tracingFactory; + + @BeforeEach + void setUp() { + spanExporter = InMemorySpanExporter.create(); + tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + openTelemetrySdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + tracer = openTelemetrySdk.getTracer("test-tracer"); + tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk).withContext(TRACER_CONTEXT); + } + + @AfterEach + void tearDown() { + tracerProvider.close(); + } + + @Test + void testAttemptSpan_linkedToParentContextFromCallingThread() { + Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); + ApiTracer apiTracer; + try (Scope scope = parentSpan.makeCurrent()) { + apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + } + + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + parentSpan.end(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(2); + + SpanData attemptSpan = + finishedSpans.stream() + .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData rootSpan = + finishedSpans.stream() + .filter(s -> s.getName().equals("application-parent-operation")) + .findFirst() + .orElseThrow(() -> new AssertionError("Parent span not found")); + + assertThat(attemptSpan.getKind()).isEqualTo(SpanKind.CLIENT); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(rootSpan.getSpanContext().getSpanId()); + assertThat(attemptSpan.getSpanContext().getTraceId()) + .isEqualTo(rootSpan.getSpanContext().getTraceId()); + } + + @Test + void testAttemptSpan_withoutParentContext_hasNoParent() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(1); + + SpanData attemptSpan = finishedSpans.get(0); + assertThat(attemptSpan.getName()).isEqualTo(FULL_METHOD_NAME); + assertThat(attemptSpan.getParentSpanContext().isValid()).isFalse(); + } + + @Test + void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { + Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); + ApiTracer apiTracer; + try (Scope scope = parentSpan.makeCurrent()) { + apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + } + + // Start attempt 0 (e.g. transient failure without explicit endAttempt before retry) + apiTracer.attemptStarted(new Object(), 0); + + // Start attempt 1 - should automatically end attempt 0 + apiTracer.attemptStarted(new Object(), 1); + assertThat(spanExporter.getFinishedSpanItems()).hasSize(1); + SpanData attempt0Span = spanExporter.getFinishedSpanItems().get(0); + assertThat(attempt0Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + + // Complete attempt 1 and operation + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + parentSpan.end(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, parent + + SpanData attempt1Span = + finishedSpans.stream() + .filter( + s -> + s.getName().equals(FULL_METHOD_NAME) + && !s.getSpanContext() + .getSpanId() + .equals(attempt0Span.getSpanContext().getSpanId())) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt 1 span not found")); + + assertThat(attempt1Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat(attempt1Span.getSpanContext().getTraceId()) + .isEqualTo(parentSpan.getSpanContext().getTraceId()); + } + + @Test + void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { + Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); + ApiTracer apiTracer; + try (Scope scope = parentSpan.makeCurrent()) { + apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + } + + apiTracer.attemptStarted(new Object(), 0); + // Operation fails while attempt was still in flight + apiTracer.operationFailed(new RuntimeException("network timeout")); + parentSpan.end(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(2); + + SpanData attemptSpan = + finishedSpans.stream() + .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt span not found")); + + assertThat(attemptSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat( + attemptSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isEqualTo("network timeout"); + } + + @Test + void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + apiTracer.operationSucceeded(); + + // Any attempts started after operation completed must be ignored + apiTracer.attemptStarted(new Object(), 0); + + assertThat(spanExporter.getFinishedSpanItems()).isEmpty(); + } +} From 2d5db0732c47d54d31911276a7c5c489be7c994b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 15:24:52 -0400 Subject: [PATCH 05/58] chore(gax): add explanatory comments to OpenTelemetryTracingTracer --- .../api/gax/tracing/OpenTelemetryTracingTracer.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 03a0b8c2a67d..b7f5fff79938 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -53,9 +53,14 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final Map attemptAttributes; private final String attemptSpanName; private final ApiTracerContext apiTracerContext; + // Captures the active trace context from the calling thread at RPC initiation. + // This allows attempt spans—including retries dispatched on background threads—to + // link back to the original parent trace. private final io.opentelemetry.context.Context parentContext; + // Lock coordinates attempt transitions and operation completion across threads. private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; + // attemptSpan is volatile to ensure fresh reads for thread-safe snapshotting. private volatile @Nullable Span attemptSpan; @Override @@ -133,9 +138,12 @@ public void attemptStarted(Object request, int attemptNumber) { Span oldSpan = null; lock.lock(); try { + // Prevent creating new attempt spans if the overall operation has already concluded. if (operationCompleted) { return; } + // If a previous attempt was not explicitly closed before a retry started, + // capture it so it can be ended cleanly outside the lock without blocking. if (attemptSpan != null) { oldSpan = attemptSpan; attemptSpan = null; @@ -168,6 +176,7 @@ public void attemptStarted(Object request, int attemptNumber) { } finally { lock.unlock(); } + // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { endAttemptSpan(oldSpan, null); } @@ -195,6 +204,8 @@ public void attemptSucceeded() { @Override public void responseHeadersReceived(java.util.Map headers) { + // Snapshot to a local variable to prevent race conditions if another thread + // clears attemptSpan concurrently. Span currentSpan = attemptSpan; if (currentSpan == null) { return; @@ -296,6 +307,8 @@ private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { @Override public void requestUrlResolved(String url) { + // Snapshot to a local variable to prevent race conditions if another thread + // clears attemptSpan concurrently. Span currentSpan = attemptSpan; if (currentSpan == null) { return; From ed0e4432fdd962b714a472f1c95181731e55761d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 15:54:51 -0400 Subject: [PATCH 06/58] chore(gax): document new and overridden methods in OpenTelemetryTracingTracer --- .../tracing/OpenTelemetryTracingTracer.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index b7f5fff79938..e2019e90aed8 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -182,16 +182,35 @@ public void attemptStarted(Object request, int attemptNumber) { } } + /** + * Signals that the overall logical operation succeeded. + * + *

Marks the operation as complete and closes any remaining in-flight attempt span. + */ @Override public void operationSucceeded() { recordErrorAndEndAttempt(null, true); } + /** + * Signals that the overall logical operation was cancelled. + * + *

Marks the operation as complete and closes any remaining in-flight attempt span with a + * {@link CancellationException}. + */ @Override public void operationCancelled() { recordErrorAndEndAttempt(new CancellationException(), true); } + /** + * Signals that the overall logical operation failed permanently. + * + *

Marks the operation as complete and closes any remaining in-flight attempt span with the + * provided error details. + * + * @param error the cause of the operation failure + */ @Override public void operationFailed(Throwable error) { recordErrorAndEndAttempt(error, true); @@ -271,6 +290,13 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error) { recordErrorAndEndAttempt(error, false); } + /** + * Records error details and ends the current attempt span in a thread-safe manner. + * + * @param error the exception associated with the attempt failure, or {@code null} if successful + * @param isOperationComplete {@code true} if this call marks the end of the entire logical + * operation, preventing subsequent retry attempts from starting + */ private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) { Span localAttemptSpan; lock.lock(); @@ -290,6 +316,15 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOpera endAttemptSpan(localAttemptSpan, error); } + /** + * Attaches response status attributes and error messages to the attempt span and ends it. + * + *

This method runs outside of synchronization locks to avoid blocking threads during + * OpenTelemetry span completion callbacks. + * + * @param localAttemptSpan the attempt span to finish + * @param error the exception that caused the attempt to end, or {@code null} if successful + */ private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { Map responseAttributes = ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); From e15ae2dd954eaa971f696bb104638838192f128e Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 16:37:36 -0400 Subject: [PATCH 07/58] fix(gax): add opentelemetry-sdk-trace test dependency in maven and bazel --- sdk-platform-java/gax-java/dependencies.properties | 1 + sdk-platform-java/gax-java/gax/BUILD.bazel | 1 + sdk-platform-java/gax-java/gax/pom.xml | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/sdk-platform-java/gax-java/dependencies.properties b/sdk-platform-java/gax-java/dependencies.properties index 6682d09eff89..135749d324d9 100644 --- a/sdk-platform-java/gax-java/dependencies.properties +++ b/sdk-platform-java/gax-java/dependencies.properties @@ -97,6 +97,7 @@ maven.io_opentelemetry_opentelemetry_sdk_testing=io.opentelemetry:opentelemetry- maven.io_opentelemetry_opentelemetry_sdk=io.opentelemetry:opentelemetry-sdk:1.57.0 maven.io_opentelemetry_opentelemetry_sdk_common=io.opentelemetry:opentelemetry-sdk-common:1.57.0 maven.io_opentelemetry_opentelemetry_sdk_metrics=io.opentelemetry:opentelemetry-sdk-metrics:1.57.0 +maven.io_opentelemetry_opentelemetry_sdk_trace=io.opentelemetry:opentelemetry-sdk-trace:1.57.0 maven.com_google_guava_guava_testlib=com.google.guava:guava-testlib:32.1.3-jre maven.org_awaitility_awaitility=org.awaitility:awaitility:4.3.0 diff --git a/sdk-platform-java/gax-java/gax/BUILD.bazel b/sdk-platform-java/gax-java/gax/BUILD.bazel index c98939ba7e18..a77329ede135 100644 --- a/sdk-platform-java/gax-java/gax/BUILD.bazel +++ b/sdk-platform-java/gax-java/gax/BUILD.bazel @@ -49,6 +49,7 @@ _TEST_COMPILE_DEPS = [ "@io_opentelemetry_opentelemetry_sdk//jar", "@io_opentelemetry_opentelemetry_sdk_metrics//jar", "@io_opentelemetry_opentelemetry_sdk_common//jar", + "@io_opentelemetry_opentelemetry_sdk_trace//jar", "@com_google_guava_guava_testlib//jar", "@org_awaitility_awaitility//jar", ] diff --git a/sdk-platform-java/gax-java/gax/pom.xml b/sdk-platform-java/gax-java/gax/pom.xml index 02c64480defb..8db559aabf12 100644 --- a/sdk-platform-java/gax-java/gax/pom.xml +++ b/sdk-platform-java/gax-java/gax/pom.xml @@ -104,6 +104,11 @@ opentelemetry-sdk-common test + + io.opentelemetry + opentelemetry-sdk-trace + test + org.junit.jupiter junit-jupiter-api From 7ba64dc2f0e56bf57b125bb2cfc9e45110469ba6 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Tue, 29 Sep 2026 17:10:59 -0400 Subject: [PATCH 08/58] feat(gax): implement operation-level client request spans (T3) --- .../tracing/OpenTelemetryTracingTracer.java | 165 ++++++++++++++---- ...OpenTelemetryTracingTracerFactoryTest.java | 8 +- ...TelemetryTracingTracerIntegrationTest.java | 67 +++++-- .../OpenTelemetryTracingTracerTest.java | 66 +++++++ 4 files changed, 254 insertions(+), 52 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index e2019e90aed8..70f3f7298869 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -35,6 +35,7 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import java.util.HashMap; import java.util.Map; @@ -52,22 +53,27 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final Tracer tracer; private final Map attemptAttributes; private final String attemptSpanName; + private final String operationSpanName; private final ApiTracerContext apiTracerContext; // Captures the active trace context from the calling thread at RPC initiation. - // This allows attempt spans—including retries dispatched on background threads—to - // link back to the original parent trace. + // This allows the operation span and attempt spans to link back to the caller's trace. private final io.opentelemetry.context.Context parentContext; + // Trace context containing the operationSpan, serving as the parent for attempt spans. + private final io.opentelemetry.context.Context operationContext; // Lock coordinates attempt transitions and operation completion across threads. private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; - // attemptSpan is volatile to ensure fresh reads for thread-safe snapshotting. + // operationSpan and attemptSpan are volatile to ensure fresh reads for thread-safe snapshotting. + private volatile @Nullable Span operationSpan; private volatile @Nullable Span attemptSpan; @Override public void injectTraceContext(java.util.Map carrier) { - if (attemptSpan != null) { + Span currentAttempt = attemptSpan; + Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; + if (spanToInject != null) { io.opentelemetry.context.Context context = - io.opentelemetry.context.Context.current().with(attemptSpan); + io.opentelemetry.context.Context.current().with(spanToInject); io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance() .inject( context, @@ -80,6 +86,17 @@ public void injectTraceContext(java.util.Map carrier) { } } + @Override + public Scope inScope() { + Span currentAttempt = attemptSpan; + Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan; + if (currentSpan == null) { + return () -> {}; + } + io.opentelemetry.context.Scope otelScope = currentSpan.makeCurrent(); + return otelScope::close; + } + /** * Creates a new instance of {@code OpenTelemetryTracingTracer}. * @@ -87,12 +104,7 @@ public void injectTraceContext(java.util.Map carrier) { * @param apiTracerContext the {@link ApiTracerContext} to use for recording spans */ OpenTelemetryTracingTracer(Tracer tracer, ApiTracerContext apiTracerContext) { - this.tracer = tracer; - this.apiTracerContext = apiTracerContext; - this.attemptSpanName = resolveAttemptSpanName(apiTracerContext); - this.attemptAttributes = new HashMap<>(); - this.parentContext = io.opentelemetry.context.Context.current(); - buildAttributes(); + this(tracer, apiTracerContext, resolveAttemptSpanName(apiTracerContext)); } /** @@ -106,12 +118,64 @@ public void injectTraceContext(java.util.Map carrier) { @InternalApi OpenTelemetryTracingTracer( Tracer tracer, ApiTracerContext apiTracerContext, String attemptSpanName) { + this(tracer, apiTracerContext, attemptSpanName, resolveOperationSpanName(attemptSpanName)); + } + + /** + * Creates a new instance of {@code OpenTelemetryTracingTracer} with explicitly provided attempt + * and operation span names. + * + * @param tracer the {@link Tracer} to use for recording spans + * @param apiTracerContext the {@link ApiTracerContext} to use for recording spans + * @param attemptSpanName the name of the individual attempt spans + * @param operationSpanName the name of the overall client request operation span + */ + @InternalApi + OpenTelemetryTracingTracer( + Tracer tracer, + ApiTracerContext apiTracerContext, + String attemptSpanName, + String operationSpanName) { this.tracer = tracer; - this.attemptSpanName = attemptSpanName; this.apiTracerContext = apiTracerContext; + this.attemptSpanName = attemptSpanName; + this.operationSpanName = operationSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); + this.operationSpan = startOperationSpan(); + this.operationContext = parentContext.with(this.operationSpan); + } + + /** + * Starts and initializes the operation-level client request span (T3). + * + * @return the newly started {@link Span} for the overall operation + */ + private Span startOperationSpan() { + SpanBuilder operationSpanBuilder = tracer.spanBuilder(operationSpanName); + operationSpanBuilder.setSpanKind(SpanKind.INTERNAL); + operationSpanBuilder.setParent(parentContext); + operationSpanBuilder.setAllAttributes( + ObservabilityUtils.toOtelAttributes(this.attemptAttributes)); + return operationSpanBuilder.startSpan(); + } + + /** + * Derives the operation-level span name from the attempt span name. + * + * @param attemptSpanName the attempt span name + * @return the operation span name + */ + private static String resolveOperationSpanName(String attemptSpanName) { + if (!Strings.isNullOrEmpty(attemptSpanName)) { + if (attemptSpanName.endsWith("/attempt")) { + String name = attemptSpanName.substring(0, attemptSpanName.length() - "/attempt".length()); + return name.isEmpty() ? "operation" : name; + } + return "attempt".equals(attemptSpanName) ? "operation" : attemptSpanName; + } + return "operation"; } private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) { @@ -139,7 +203,7 @@ public void attemptStarted(Object request, int attemptNumber) { lock.lock(); try { // Prevent creating new attempt spans if the overall operation has already concluded. - if (operationCompleted) { + if (operationCompleted || operationSpan == null) { return; } // If a previous attempt was not explicitly closed before a retry started, @@ -166,8 +230,8 @@ public void attemptStarted(Object request, int attemptNumber) { // Attempt spans are of the CLIENT kind spanBuilder.setSpanKind(SpanKind.CLIENT); - // Link attempt span to parent context - spanBuilder.setParent(parentContext); + // Link attempt span to operation context (parent T3 span) + spanBuilder.setParent(operationContext); // Pass the combined attributes to the new SpanBuilder method spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); @@ -185,35 +249,79 @@ public void attemptStarted(Object request, int attemptNumber) { /** * Signals that the overall logical operation succeeded. * - *

Marks the operation as complete and closes any remaining in-flight attempt span. + *

Closes any remaining in-flight attempt span and ends the operation span. */ @Override public void operationSucceeded() { - recordErrorAndEndAttempt(null, true); + recordErrorAndEndOperation(null); } /** * Signals that the overall logical operation was cancelled. * - *

Marks the operation as complete and closes any remaining in-flight attempt span with a - * {@link CancellationException}. + *

Closes any remaining in-flight attempt span with a {@link CancellationException} and ends + * the operation span with an ERROR status. */ @Override public void operationCancelled() { - recordErrorAndEndAttempt(new CancellationException(), true); + recordErrorAndEndOperation(new CancellationException()); } /** * Signals that the overall logical operation failed permanently. * - *

Marks the operation as complete and closes any remaining in-flight attempt span with the - * provided error details. + *

Closes any remaining in-flight attempt span and ends the operation span with the provided + * error details and an ERROR status. * * @param error the cause of the operation failure */ @Override public void operationFailed(Throwable error) { - recordErrorAndEndAttempt(error, true); + recordErrorAndEndOperation(error); + } + + /** + * Records error details and ends both the active attempt span and the operation span in a + * thread-safe manner. + * + * @param error the exception associated with the operation failure, or {@code null} if successful + */ + private void recordErrorAndEndOperation(@Nullable Throwable error) { + Span localOperationSpan; + Span localAttemptSpan; + lock.lock(); + try { + operationCompleted = true; + localOperationSpan = operationSpan; + if (localOperationSpan == null) { + return; + } + operationSpan = null; + localAttemptSpan = attemptSpan; + attemptSpan = null; + } finally { + lock.unlock(); + } + + if (localAttemptSpan != null) { + endAttemptSpan(localAttemptSpan, error); + } + + Map responseAttributes = + ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); + if (!responseAttributes.isEmpty()) { + localOperationSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); + } + + if (error != null) { + localOperationSpan.setStatus(StatusCode.ERROR); + if (!Strings.isNullOrEmpty(error.getMessage())) { + localOperationSpan.setAttribute( + ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); + } + } + + localOperationSpan.end(); } @Override @@ -286,24 +394,15 @@ public void attemptPermanentFailure(Throwable error) { recordErrorAndEndAttempt(error); } - private void recordErrorAndEndAttempt(@Nullable Throwable error) { - recordErrorAndEndAttempt(error, false); - } - /** * Records error details and ends the current attempt span in a thread-safe manner. * * @param error the exception associated with the attempt failure, or {@code null} if successful - * @param isOperationComplete {@code true} if this call marks the end of the entire logical - * operation, preventing subsequent retry attempts from starting */ - private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) { + private void recordErrorAndEndAttempt(@Nullable Throwable error) { Span localAttemptSpan; lock.lock(); try { - if (isOperationComplete) { - operationCompleted = true; - } localAttemptSpan = attemptSpan; if (localAttemptSpan == null) { return; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java index de7941522587..4a58a3b07bfd 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java @@ -229,7 +229,7 @@ void testNewTracer_withContext_grpc_usesFullMethodName() { tracerInstance.attemptStarted(null, 1); - verify(tracer).spanBuilder("google.cloud.v1.Service/Method"); + verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service/Method"); } @ParameterizedTest @@ -256,7 +256,7 @@ void testNewTracer_withContext_http_usesHttpMethodAndPathTemplate( tracerInstance.attemptStarted(null, 1); - verify(tracer).spanBuilder(expectedSpanName); + verify(tracer, atLeastOnce()).spanBuilder(expectedSpanName); } @Test @@ -274,7 +274,7 @@ void testNewTracer_withContext_http_noHttpMethodOrPathTemplate_usesFullMethodNam tracerInstance.attemptStarted(null, 1); - verify(tracer).spanBuilder("google.cloud.v1.Service.Method"); + verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service.Method"); } @Test @@ -310,7 +310,7 @@ void testNewTracer_mergesFactoryContext() { tracerInstance.attemptStarted(null, 1); ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Attributes.class); - verify(spanBuilder).setAllAttributes(attributesCaptor.capture()); + verify(spanBuilder, atLeastOnce()).setAllAttributes(attributesCaptor.capture()); Attributes attributes = attributesCaptor.getValue(); assertThat(attributes.asMap()) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 31832fbc3d71..613770ec84ed 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -103,13 +103,18 @@ void testAttemptSpan_linkedToParentContextFromCallingThread() { parentSpan.end(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(2); + assertThat(finishedSpans).hasSize(3); // root, operation, attempt SpanData attemptSpan = finishedSpans.stream() - .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .filter(s -> s.getKind() == SpanKind.CLIENT) .findFirst() .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); SpanData rootSpan = finishedSpans.stream() .filter(s -> s.getName().equals("application-parent-operation")) @@ -117,9 +122,12 @@ void testAttemptSpan_linkedToParentContextFromCallingThread() { .orElseThrow(() -> new AssertionError("Parent span not found")); assertThat(attemptSpan.getKind()).isEqualTo(SpanKind.CLIENT); - assertThat(attemptSpan.getParentSpanId()).isEqualTo(rootSpan.getSpanContext().getSpanId()); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + assertThat(operationSpan.getParentSpanId()).isEqualTo(rootSpan.getSpanContext().getSpanId()); assertThat(attemptSpan.getSpanContext().getTraceId()) .isEqualTo(rootSpan.getSpanContext().getTraceId()); + assertThat(operationSpan.getSpanContext().getTraceId()) + .isEqualTo(rootSpan.getSpanContext().getTraceId()); } @Test @@ -131,11 +139,21 @@ void testAttemptSpan_withoutParentContext_hasNoParent() { apiTracer.operationSucceeded(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(1); + assertThat(finishedSpans).hasSize(2); // operation, attempt - SpanData attemptSpan = finishedSpans.get(0); - assertThat(attemptSpan.getName()).isEqualTo(FULL_METHOD_NAME); - assertThat(attemptSpan.getParentSpanContext().isValid()).isFalse(); + SpanData attemptSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + assertThat(operationSpan.getParentSpanContext().isValid()).isFalse(); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); } @Test @@ -153,7 +171,6 @@ void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { apiTracer.attemptStarted(new Object(), 1); assertThat(spanExporter.getFinishedSpanItems()).hasSize(1); SpanData attempt0Span = spanExporter.getFinishedSpanItems().get(0); - assertThat(attempt0Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); // Complete attempt 1 and operation apiTracer.attemptSucceeded(); @@ -161,20 +178,31 @@ void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { parentSpan.end(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, parent + assertThat(finishedSpans).hasSize(4); // attempt 0, attempt 1, operation, parent + + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + assertThat(attempt0Span.getParentSpanId()) + .isEqualTo(operationSpan.getSpanContext().getSpanId()); SpanData attempt1Span = finishedSpans.stream() .filter( s -> - s.getName().equals(FULL_METHOD_NAME) + s.getKind() == SpanKind.CLIENT && !s.getSpanContext() .getSpanId() .equals(attempt0Span.getSpanContext().getSpanId())) .findFirst() .orElseThrow(() -> new AssertionError("Attempt 1 span not found")); - assertThat(attempt1Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat(attempt1Span.getParentSpanId()) + .isEqualTo(operationSpan.getSpanContext().getSpanId()); + assertThat(operationSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); assertThat(attempt1Span.getSpanContext().getTraceId()) .isEqualTo(parentSpan.getSpanContext().getTraceId()); } @@ -193,15 +221,21 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { parentSpan.end(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(2); + assertThat(finishedSpans).hasSize(3); // parent, operation, attempt SpanData attemptSpan = finishedSpans.stream() - .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .filter(s -> s.getKind() == SpanKind.CLIENT) .findFirst() .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); - assertThat(attemptSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + assertThat(operationSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); assertThat( attemptSpan .getAttributes() @@ -218,6 +252,9 @@ void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { // Any attempts started after operation completed must be ignored apiTracer.attemptStarted(new Object(), 0); - assertThat(spanExporter.getFinishedSpanItems()).isEmpty(); + // Only the operation span was emitted and ended + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(1); + assertThat(finishedSpans.get(0).getKind()).isEqualTo(SpanKind.INTERNAL); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 2bd2dd320ce4..7ccb6cde0840 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -66,6 +66,8 @@ class OpenTelemetryTracingTracerTest { @Mock private Tracer tracer; @Mock private SpanBuilder spanBuilder; @Mock private Span span; + @Mock private SpanBuilder operationSpanBuilder; + @Mock private Span operationSpan; private OpenTelemetryTracingTracer openTelemetryTracingTracer; private static final String ATTEMPT_SPAN_NAME = "Service/Method/attempt"; @@ -76,6 +78,20 @@ void setUp() { lenient().when(spanBuilder.setParent(any())).thenReturn(spanBuilder); lenient().when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); lenient().when(spanBuilder.startSpan()).thenReturn(span); + + lenient() + .when(operationSpanBuilder.setSpanKind(any(SpanKind.class))) + .thenReturn(operationSpanBuilder); + lenient().when(operationSpanBuilder.setParent(any())).thenReturn(operationSpanBuilder); + lenient() + .when(operationSpanBuilder.setAllAttributes(any(Attributes.class))) + .thenReturn(operationSpanBuilder); + lenient().when(operationSpanBuilder.startSpan()).thenReturn(operationSpan); + lenient() + .when(operationSpan.storeInContext(any(io.opentelemetry.context.Context.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + lenient().when(tracer.spanBuilder("Service/Method")).thenReturn(operationSpanBuilder); + openTelemetryTracingTracer = new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); } @@ -696,6 +712,7 @@ void testOperationSucceeded_endsActiveAttemptSpan() { openTelemetryTracingTracer.operationSucceeded(); verify(span).end(); + verify(operationSpan).end(); } @Test @@ -705,6 +722,8 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); verify(span).end(); + verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).end(); } @Test @@ -715,6 +734,8 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); verify(span).setAllAttributes(attrsCaptor.capture()); verify(span).end(); + verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).end(); assertThat(attrsCaptor.getValue().asMap()) .containsEntry( @@ -722,6 +743,51 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { "CANCELLED"); } + @Test + void testInScope_withAttemptSpan() { + io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); + when(span.makeCurrent()).thenReturn(mockScope); + + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + try (ApiTracer.Scope scope = openTelemetryTracingTracer.inScope()) { + verify(span).makeCurrent(); + } + verify(mockScope).close(); + } + + @Test + void testInScope_withOperationSpanFallback() { + io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); + when(operationSpan.makeCurrent()).thenReturn(mockScope); + + try (ApiTracer.Scope scope = openTelemetryTracingTracer.inScope()) { + verify(operationSpan).makeCurrent(); + } + verify(mockScope).close(); + } + + @Test + void testInjectTraceContext_withOperationSpanFallback() { + io.opentelemetry.api.trace.SpanContext mockSpanContext = + io.opentelemetry.api.trace.SpanContext.create( + "00000000000000000000000000000003", + "0000000000000004", + io.opentelemetry.api.trace.TraceFlags.getSampled(), + io.opentelemetry.api.trace.TraceState.getDefault()); + Span realSpan = Span.wrap(mockSpanContext); + when(operationSpanBuilder.startSpan()).thenReturn(realSpan); + + openTelemetryTracingTracer = + new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); + + Map carrier = new java.util.HashMap<>(); + openTelemetryTracingTracer.injectTraceContext(carrier); + + assertThat(carrier).containsKey("traceparent"); + assertThat(carrier.get("traceparent")).contains("00000000000000000000000000000003"); + assertThat(carrier.get("traceparent")).contains("0000000000000004"); + } + @Test void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { Span span1 = mock(Span.class); From 349c14af70b97f4d24928dda21db1e2c7a7bc00e Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 11:34:09 -0400 Subject: [PATCH 09/58] fix(gax): address review feedback on span names and span completion helper --- .../tracing/OpenTelemetryTracingTracer.java | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 70f3f7298869..f4018f6175d5 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -138,8 +138,9 @@ public Scope inScope() { String operationSpanName) { this.tracer = tracer; this.apiTracerContext = apiTracerContext; - this.attemptSpanName = attemptSpanName; this.operationSpanName = operationSpanName; + this.attemptSpanName = + attemptSpanName.equals(operationSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); @@ -242,7 +243,7 @@ public void attemptStarted(Object request, int attemptNumber) { } // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { - endAttemptSpan(oldSpan, null); + endSpan(oldSpan, null); } } @@ -304,24 +305,10 @@ private void recordErrorAndEndOperation(@Nullable Throwable error) { } if (localAttemptSpan != null) { - endAttemptSpan(localAttemptSpan, error); + endSpan(localAttemptSpan, error); } - Map responseAttributes = - ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); - if (!responseAttributes.isEmpty()) { - localOperationSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); - } - - if (error != null) { - localOperationSpan.setStatus(StatusCode.ERROR); - if (!Strings.isNullOrEmpty(error.getMessage())) { - localOperationSpan.setAttribute( - ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); - } - } - - localOperationSpan.end(); + endSpan(localOperationSpan, error); } @Override @@ -412,31 +399,33 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error) { lock.unlock(); } - endAttemptSpan(localAttemptSpan, error); + endSpan(localAttemptSpan, error); } /** - * Attaches response status attributes and error messages to the attempt span and ends it. + * Attaches response status attributes and error messages to the span and ends it. * *

This method runs outside of synchronization locks to avoid blocking threads during * OpenTelemetry span completion callbacks. * - * @param localAttemptSpan the attempt span to finish - * @param error the exception that caused the attempt to end, or {@code null} if successful + * @param span the span to finish + * @param error the exception that caused the span to end, or {@code null} if successful */ - private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { + private void endSpan(Span span, @Nullable Throwable error) { Map responseAttributes = ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); if (!responseAttributes.isEmpty()) { - localAttemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); + span.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); } - if (error != null && !Strings.isNullOrEmpty(error.getMessage())) { - localAttemptSpan.setAttribute( - ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); + if (error != null) { + span.setStatus(StatusCode.ERROR); + if (!Strings.isNullOrEmpty(error.getMessage())) { + span.setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); + } } - localAttemptSpan.end(); + span.end(); } @Override From b542ff110ca65fe86f92132ad892a4e06f87edd1 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:15:00 -0400 Subject: [PATCH 10/58] fix(gax): avoid potential NPE when resolving attempt span name --- .../com/google/api/gax/tracing/OpenTelemetryTracingTracer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index f4018f6175d5..e08dd54373ed 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -140,7 +140,7 @@ public Scope inScope() { this.apiTracerContext = apiTracerContext; this.operationSpanName = operationSpanName; this.attemptSpanName = - attemptSpanName.equals(operationSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; + operationSpanName.equals(attemptSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); From 23b22fcb8db7887c6a43976da3de7804e87a8589 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:33:33 -0400 Subject: [PATCH 11/58] fix(tracing): record CancellationException on lingering attempt when superseded by retry --- .../com/google/api/gax/tracing/OpenTelemetryTracingTracer.java | 2 +- .../google/api/gax/tracing/OpenTelemetryTracingTracerTest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index e08dd54373ed..1bd18b0faaee 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -243,7 +243,7 @@ public void attemptStarted(Object request, int attemptNumber) { } // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { - endSpan(oldSpan, null); + endSpan(oldSpan, new CancellationException("Attempt superseded by retry")); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 7ccb6cde0840..1e6ca3180af2 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -799,6 +799,7 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { // Start a second attempt before the first attempt was ended openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(span1).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); verify(span1).end(); verify(span2, never()).end(); From 018b3d80ec213a3c3118df590bb0244bd17be2c6 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:37:28 -0400 Subject: [PATCH 12/58] style(tracing): use simple names Map and HashMap in OpenTelemetryTracingTracer --- .../google/api/gax/tracing/OpenTelemetryTracingTracer.java | 4 ++-- .../api/gax/tracing/OpenTelemetryTracingTracerTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 1bd18b0faaee..baf5c31695c0 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -68,7 +68,7 @@ class OpenTelemetryTracingTracer implements ApiTracer { private volatile @Nullable Span attemptSpan; @Override - public void injectTraceContext(java.util.Map carrier) { + public void injectTraceContext(Map carrier) { Span currentAttempt = attemptSpan; Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; if (spanToInject != null) { @@ -317,7 +317,7 @@ public void attemptSucceeded() { } @Override - public void responseHeadersReceived(java.util.Map headers) { + public void responseHeadersReceived(Map headers) { // Snapshot to a local variable to prevent race conditions if another thread // clears attemptSpan concurrently. Span currentSpan = attemptSpan; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 1e6ca3180af2..41a56372d8b7 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -53,6 +53,7 @@ import io.opentelemetry.api.trace.Tracer; import java.net.ConnectException; import java.net.SocketTimeoutException; +import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -780,7 +781,7 @@ void testInjectTraceContext_withOperationSpanFallback() { openTelemetryTracingTracer = new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); - Map carrier = new java.util.HashMap<>(); + Map carrier = new HashMap<>(); openTelemetryTracingTracer.injectTraceContext(carrier); assertThat(carrier).containsKey("traceparent"); From 4cab2e1fda7140a10a3c93cd33895c9309850ce8 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:45:55 -0400 Subject: [PATCH 13/58] style(tracing): use simple name StatusCode in OpenTelemetryTracingTracerTest --- .../api/gax/tracing/OpenTelemetryTracingTracerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 41a56372d8b7..abeda4771d68 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -50,6 +50,7 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -723,7 +724,7 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); verify(span).end(); - verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).setStatus(StatusCode.ERROR); verify(operationSpan).end(); } @@ -735,7 +736,7 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); verify(span).setAllAttributes(attrsCaptor.capture()); verify(span).end(); - verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).setStatus(StatusCode.ERROR); verify(operationSpan).end(); assertThat(attrsCaptor.getValue().asMap()) @@ -800,7 +801,7 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { // Start a second attempt before the first attempt was ended openTelemetryTracingTracer.attemptStarted(new Object(), 1); - verify(span1).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(span1).setStatus(StatusCode.ERROR); verify(span1).end(); verify(span2, never()).end(); From 8274051031089376af05f1b94e2f7aaa081dd457 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:50:54 -0400 Subject: [PATCH 14/58] fix(tracing): end superseded attempt span without error status --- .../com/google/api/gax/tracing/OpenTelemetryTracingTracer.java | 2 +- .../google/api/gax/tracing/OpenTelemetryTracingTracerTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index baf5c31695c0..66b8d8ed3c77 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -243,7 +243,7 @@ public void attemptStarted(Object request, int attemptNumber) { } // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { - endSpan(oldSpan, new CancellationException("Attempt superseded by retry")); + endSpan(oldSpan, null); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index abeda4771d68..556845cab1a5 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -801,7 +801,6 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { // Start a second attempt before the first attempt was ended openTelemetryTracingTracer.attemptStarted(new Object(), 1); - verify(span1).setStatus(StatusCode.ERROR); verify(span1).end(); verify(span2, never()).end(); From 1e66751311517ce2cedc5d09936a7e8202edac29 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:57:25 -0400 Subject: [PATCH 15/58] style(tracing): use simple names Context and W3CTraceContextPropagator in OpenTelemetryTracingTracer --- .../api/gax/tracing/OpenTelemetryTracingTracer.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 66b8d8ed3c77..0e1444c52559 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -37,6 +37,8 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CancellationException; @@ -57,9 +59,9 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final ApiTracerContext apiTracerContext; // Captures the active trace context from the calling thread at RPC initiation. // This allows the operation span and attempt spans to link back to the caller's trace. - private final io.opentelemetry.context.Context parentContext; + private final Context parentContext; // Trace context containing the operationSpan, serving as the parent for attempt spans. - private final io.opentelemetry.context.Context operationContext; + private final Context operationContext; // Lock coordinates attempt transitions and operation completion across threads. private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; @@ -72,9 +74,8 @@ public void injectTraceContext(Map carrier) { Span currentAttempt = attemptSpan; Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; if (spanToInject != null) { - io.opentelemetry.context.Context context = - io.opentelemetry.context.Context.current().with(spanToInject); - io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance() + Context context = Context.current().with(spanToInject); + W3CTraceContextPropagator.getInstance() .inject( context, carrier, @@ -142,7 +143,7 @@ public Scope inScope() { this.attemptSpanName = operationSpanName.equals(attemptSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; this.attemptAttributes = new HashMap<>(); - this.parentContext = io.opentelemetry.context.Context.current(); + this.parentContext = Context.current(); buildAttributes(); this.operationSpan = startOperationSpan(); this.operationContext = parentContext.with(this.operationSpan); From 33b1d238898f2f0bd82a7ffdda45489f28ff7d21 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:47:59 -0400 Subject: [PATCH 16/58] style(tracing): static import OpenTelemetry StatusCode.ERROR to avoid collision with GAX StatusCode --- .../api/gax/tracing/OpenTelemetryTracingTracerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 556845cab1a5..1655e3cb69d5 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -30,6 +30,7 @@ package com.google.api.gax.tracing; import static com.google.common.truth.Truth.assertThat; +import static io.opentelemetry.api.trace.StatusCode.ERROR; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -50,7 +51,6 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -724,7 +724,7 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); verify(span).end(); - verify(operationSpan).setStatus(StatusCode.ERROR); + verify(operationSpan).setStatus(ERROR); verify(operationSpan).end(); } @@ -736,7 +736,7 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); verify(span).setAllAttributes(attrsCaptor.capture()); verify(span).end(); - verify(operationSpan).setStatus(StatusCode.ERROR); + verify(operationSpan).setStatus(ERROR); verify(operationSpan).end(); assertThat(attrsCaptor.getValue().asMap()) From 412f398ee9d36f906f2ffa0fd837e980f5735a8d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:26:03 -0400 Subject: [PATCH 17/58] fix(tracing): restore canonical attempt span name and suppress MustBeClosedChecker --- .../google/api/gax/tracing/OpenTelemetryTracingTracer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 0e1444c52559..98cdb0375748 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -88,6 +88,7 @@ public void injectTraceContext(Map carrier) { } @Override + @SuppressWarnings("MustBeClosedChecker") public Scope inScope() { Span currentAttempt = attemptSpan; Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan; @@ -140,8 +141,7 @@ public Scope inScope() { this.tracer = tracer; this.apiTracerContext = apiTracerContext; this.operationSpanName = operationSpanName; - this.attemptSpanName = - operationSpanName.equals(attemptSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; + this.attemptSpanName = attemptSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = Context.current(); buildAttributes(); From 4b851adb7d4ef51cab69f943da9b61f0dbf5eb51 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:27:28 -0400 Subject: [PATCH 18/58] docs(tracing): add Javadoc and explanatory comments for T3 operation spans --- .../api/gax/tracing/OpenTelemetryTracingTracer.java | 12 ++++++++++++ .../OpenTelemetryTracingTracerIntegrationTest.java | 8 ++++++++ .../gax/tracing/OpenTelemetryTracingTracerTest.java | 8 ++++++++ 3 files changed, 28 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 98cdb0375748..a4a87463c26d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -71,6 +71,9 @@ class OpenTelemetryTracingTracer implements ApiTracer { @Override public void injectTraceContext(Map carrier) { + // Prefer the active attempt span (T4) so outgoing RPC wire context reflects the specific + // attempt; + // fall back to the overall operation span (T3) if no attempt is currently in-flight. Span currentAttempt = attemptSpan; Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; if (spanToInject != null) { @@ -90,6 +93,8 @@ public void injectTraceContext(Map carrier) { @Override @SuppressWarnings("MustBeClosedChecker") public Scope inScope() { + // Attach the active attempt span to the current execution thread context; + // fall back to the overall operation span when between attempts. Span currentAttempt = attemptSpan; Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan; if (currentSpan == null) { @@ -180,6 +185,12 @@ private static String resolveOperationSpanName(String attemptSpanName) { return "operation"; } + /** + * Resolves the canonical attempt-level span name based on transport and context. + * + * @param apiTracerContext the tracer context containing transport and method metadata + * @return the attempt span name + */ private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) { if (apiTracerContext.transport() == ApiTracerContext.Transport.GRPC) { // gRPC Uses the full method name as span name. @@ -195,6 +206,7 @@ private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) } } + /** Copies attempt-level attributes from the tracer context into the local attribute cache. */ private void buildAttributes() { this.attemptAttributes.putAll(this.apiTracerContext.getAttemptAttributes()); } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 613770ec84ed..4aa3a47fe807 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -91,6 +91,8 @@ void tearDown() { @Test void testAttemptSpan_linkedToParentContextFromCallingThread() { + // Verifies that when an application trace parent exists, the operation span (T3) + // links to the application parent, and the attempt span (T4) links to the operation span. Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); ApiTracer apiTracer; try (Scope scope = parentSpan.makeCurrent()) { @@ -132,6 +134,8 @@ void testAttemptSpan_linkedToParentContextFromCallingThread() { @Test void testAttemptSpan_withoutParentContext_hasNoParent() { + // Verifies that without an external parent context, the operation span acts as root + // and the attempt span is a child of the operation span. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); apiTracer.attemptStarted(new Object(), 0); @@ -209,6 +213,8 @@ void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { @Test void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { + // Verifies that when the operation fails permanently while an attempt is in-flight, + // both the attempt span and the operation span end with the error status and message. Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); ApiTracer apiTracer; try (Scope scope = parentSpan.makeCurrent()) { @@ -245,6 +251,8 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { @Test void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { + // Verifies that after operation completion, subsequent attemptStarted calls + // do not create orphan attempt spans. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); apiTracer.operationSucceeded(); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 1655e3cb69d5..6d1659ee0c71 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -747,6 +747,7 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { @Test void testInScope_withAttemptSpan() { + // Verifies that inScope() activates the current attempt span if an attempt is currently active. io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); when(span.makeCurrent()).thenReturn(mockScope); @@ -759,6 +760,8 @@ void testInScope_withAttemptSpan() { @Test void testInScope_withOperationSpanFallback() { + // Verifies that inScope() falls back to activating the operation span when no attempt span is + // active. io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); when(operationSpan.makeCurrent()).thenReturn(mockScope); @@ -770,6 +773,8 @@ void testInScope_withOperationSpanFallback() { @Test void testInjectTraceContext_withOperationSpanFallback() { + // Verifies that injectTraceContext() injects the operation span context into the carrier + // when between attempts so that context propagation doesn't drop trace state. io.opentelemetry.api.trace.SpanContext mockSpanContext = io.opentelemetry.api.trace.SpanContext.create( "00000000000000000000000000000003", @@ -792,6 +797,8 @@ void testInjectTraceContext_withOperationSpanFallback() { @Test void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { + // Verifies that starting a new retry attempt cleanly closes any lingering previous attempt + // span. Span span1 = mock(Span.class); Span span2 = mock(Span.class); @@ -811,6 +818,7 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { @Test void testAttemptStarted_afterOperationCompleted_doesNotStartNewSpan() { + // Verifies that after operation completion, late callbacks cannot spawn new attempt spans. openTelemetryTracingTracer.operationSucceeded(); // Attempting to start a new attempt after operation completion should be a no-op From ff8e44831511ed8f3c7830aa9ee3a53b798cf739 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 11:39:52 -0400 Subject: [PATCH 19/58] feat(gax): verify T3/T4 span hierarchy and retry aggregation --- ...TelemetryTracingTracerIntegrationTest.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 4aa3a47fe807..71b92a56d3e9 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -44,7 +44,9 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; import java.util.List; +import java.util.stream.Collectors; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -265,4 +267,97 @@ void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { assertThat(finishedSpans).hasSize(1); assertThat(finishedSpans.get(0).getKind()).isEqualTo(SpanKind.INTERNAL); } + + @Test + void testRetrySucceeds_operationAggregatesSuccessAttributes() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + // Attempt 0 fails with transient error + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptFailedDuration(new RuntimeException("transient 503"), Duration.ofMillis(10)); + + // Attempt 1 succeeds + apiTracer.attemptStarted(new Object(), 1); + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation + + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + List attemptSpans = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .collect(Collectors.toList()); + assertThat(attemptSpans).hasSize(2); + + for (SpanData attempt : attemptSpans) { + assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + } + + assertThat(operationSpan.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("OK"); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNull(); + } + + @Test + void testRetriesExhausted_operationAggregatesFailureAttributes() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + // Attempt 0 fails with transient error + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptFailedDuration(new RuntimeException("transient 503"), Duration.ofMillis(10)); + + // Attempt 1 fails and exhausts retries + apiTracer.attemptStarted(new Object(), 1); + RuntimeException finalError = new RuntimeException("unavailable: retries exhausted"); + apiTracer.attemptFailedRetriesExhausted(finalError); + apiTracer.operationFailed(finalError); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation + + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + List attemptSpans = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .collect(Collectors.toList()); + assertThat(attemptSpans).hasSize(2); + + for (SpanData attempt : attemptSpans) { + assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + } + + assertThat(operationSpan.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isEqualTo("unavailable: retries exhausted"); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } } From 22ca25d3967ad1de8b6fdd9b78727ca4175a2007 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:43:42 -0400 Subject: [PATCH 20/58] test(tracing): explicitly verify exactly one internal operation span in retry tests --- .../OpenTelemetryTracingTracerIntegrationTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 71b92a56d3e9..50d77925e9e6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -284,11 +284,12 @@ void testRetrySucceeds_operationAggregatesSuccessAttributes() { List finishedSpans = spanExporter.getFinishedSpanItems(); assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation - SpanData operationSpan = + List internalSpans = finishedSpans.stream() .filter(s -> s.getKind() == SpanKind.INTERNAL) - .findFirst() - .orElseThrow(() -> new AssertionError("Operation span not found")); + .collect(Collectors.toList()); + assertThat(internalSpans).hasSize(1); + SpanData operationSpan = internalSpans.get(0); List attemptSpans = finishedSpans.stream() @@ -331,11 +332,12 @@ void testRetriesExhausted_operationAggregatesFailureAttributes() { List finishedSpans = spanExporter.getFinishedSpanItems(); assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation - SpanData operationSpan = + List internalSpans = finishedSpans.stream() .filter(s -> s.getKind() == SpanKind.INTERNAL) - .findFirst() - .orElseThrow(() -> new AssertionError("Operation span not found")); + .collect(Collectors.toList()); + assertThat(internalSpans).hasSize(1); + SpanData operationSpan = internalSpans.get(0); List attemptSpans = finishedSpans.stream() From f27fdb36114903a4d5bf66164178a0deb9db8ba0 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:28:18 -0400 Subject: [PATCH 21/58] docs(tracing): add explanatory comments to T3/T4 hierarchy and retry tests --- .../OpenTelemetryTracingTracerIntegrationTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 50d77925e9e6..81a5949eed91 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -270,6 +270,10 @@ void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { @Test void testRetrySucceeds_operationAggregatesSuccessAttributes() { + // Verifies that when a transient failure is retried and succeeds: + // 1. Exactly one overall INTERNAL operation span (T3) is created. + // 2. Both attempt spans (T4) have the operation span as their parent. + // 3. The operation span aggregates the successful status (OK) from the final attempt. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); // Attempt 0 fails with transient error @@ -317,6 +321,10 @@ void testRetrySucceeds_operationAggregatesSuccessAttributes() { @Test void testRetriesExhausted_operationAggregatesFailureAttributes() { + // Verifies that when retries are exhausted: + // 1. Exactly one overall INTERNAL operation span (T3) is created. + // 2. Both attempt spans (T4) have the operation span as their parent. + // 3. The operation span aggregates the final ERROR status and error attributes. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); // Attempt 0 fails with transient error From 8de17a01d584163bf775e037292bf9bbcc9afc10 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:36:26 -0400 Subject: [PATCH 22/58] refactor(tracing): extract verifySpanHierarchyAndGetOperationSpan test helper --- ...TelemetryTracingTracerIntegrationTest.java | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 81a5949eed91..b479a6a5eccb 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -285,25 +285,8 @@ void testRetrySucceeds_operationAggregatesSuccessAttributes() { apiTracer.attemptSucceeded(); apiTracer.operationSucceeded(); - List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation - - List internalSpans = - finishedSpans.stream() - .filter(s -> s.getKind() == SpanKind.INTERNAL) - .collect(Collectors.toList()); - assertThat(internalSpans).hasSize(1); - SpanData operationSpan = internalSpans.get(0); - - List attemptSpans = - finishedSpans.stream() - .filter(s -> s.getKind() == SpanKind.CLIENT) - .collect(Collectors.toList()); - assertThat(attemptSpans).hasSize(2); - - for (SpanData attempt : attemptSpans) { - assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); - } + SpanData operationSpan = + verifySpanHierarchyAndGetOperationSpan(spanExporter.getFinishedSpanItems()); assertThat(operationSpan.getStatus().getStatusCode()) .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); @@ -337,8 +320,32 @@ void testRetriesExhausted_operationAggregatesFailureAttributes() { apiTracer.attemptFailedRetriesExhausted(finalError); apiTracer.operationFailed(finalError); - List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation + SpanData operationSpan = + verifySpanHierarchyAndGetOperationSpan(spanExporter.getFinishedSpanItems()); + + assertThat(operationSpan.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isEqualTo("unavailable: retries exhausted"); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } + + /** + * Verifies that exactly one internal operation span and two client attempt spans exist in the + * finished spans, and that each attempt span has the operation span as its parent. + * + * @param finishedSpans the list of recorded finished spans + * @return the verified operation {@link SpanData} + */ + private SpanData verifySpanHierarchyAndGetOperationSpan(List finishedSpans) { + assertThat(finishedSpans).hasSize(3); List internalSpans = finishedSpans.stream() @@ -356,18 +363,6 @@ void testRetriesExhausted_operationAggregatesFailureAttributes() { for (SpanData attempt : attemptSpans) { assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); } - - assertThat(operationSpan.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); - assertThat( - operationSpan - .getAttributes() - .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) - .isEqualTo("unavailable: retries exhausted"); - assertThat( - operationSpan - .getAttributes() - .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) - .isNotNull(); + return operationSpan; } } From 8eaceafaea5637bd9d139ccd3fa5987550e903bf Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 11:42:56 -0400 Subject: [PATCH 23/58] feat(gax): correlate M3 client duration metrics with T3 span exemplars --- .../api/gax/tracing/CompositeTracer.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 118b2d89a334..d6c29758df86 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; /** * A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of @@ -57,7 +58,10 @@ public Scope inScope() { try { for (ApiTracer child : children) { - childScopes.add(child.inScope()); + Scope scope = child.inScope(); + if (scope != null) { + childScopes.add(scope); + } } } catch (RuntimeException e) { for (int i = childScopes.size() - 1; i >= 0; i--) { @@ -91,22 +95,48 @@ public Scope inScope() { @Override public void operationSucceeded() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationSucceeded(); + @Nullable Scope scope = null; + try { + scope = inScope(); + } catch (RuntimeException e) { + // Ignore to prevent disrupting the lifecycle notification + } + try (Scope s = scope) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationSucceeded(); + } } } @Override public void operationCancelled() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationCancelled(); + @Nullable Scope scope = null; + try { + scope = inScope(); + } catch (RuntimeException e) { + // Ignore to prevent disrupting the lifecycle notification + } + try (Scope s = scope) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationCancelled(); + } } } @Override public void operationFailed(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationFailed(error); + @Nullable Scope scope = null; + try { + scope = inScope(); + } catch (RuntimeException e) { + if (error != null) { + error.addSuppressed(e); + } + } + try (Scope s = scope) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationFailed(error); + } } } From 8acab46377a3ea0eedef3e2b1505bc258ce629d9 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:27:44 -0400 Subject: [PATCH 24/58] fix(tracing): avoid self-suppression in CompositeTracer lifecycle notifications --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index d6c29758df86..15ede83eef02 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -129,7 +129,7 @@ public void operationFailed(Throwable error) { try { scope = inScope(); } catch (RuntimeException e) { - if (error != null) { + if (error != null && error != e) { error.addSuppressed(e); } } From 92ae605795cdf4cd0cc4da1859596c58b436e1e0 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:31:08 -0400 Subject: [PATCH 25/58] fix(tracing): avoid mutating caller exception in CompositeTracer operationFailed --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 15ede83eef02..906b9f0d6c34 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -129,9 +129,7 @@ public void operationFailed(Throwable error) { try { scope = inScope(); } catch (RuntimeException e) { - if (error != null && error != e) { - error.addSuppressed(e); - } + // Ignore to prevent disrupting the lifecycle notification } try (Scope s = scope) { for (int i = children.size() - 1; i >= 0; i--) { From f7452926985b363a5d20e423b4443ab950204d92 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:38:10 -0400 Subject: [PATCH 26/58] refactor(tracing): extract runInScope helper in CompositeTracer --- .../api/gax/tracing/CompositeTracer.java | 51 +++++++++---------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 906b9f0d6c34..46c32e454912 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -93,8 +93,7 @@ public Scope inScope() { }; } - @Override - public void operationSucceeded() { + private void runInScope(Runnable runnable) { @Nullable Scope scope = null; try { scope = inScope(); @@ -102,40 +101,38 @@ public void operationSucceeded() { // Ignore to prevent disrupting the lifecycle notification } try (Scope s = scope) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationSucceeded(); - } + runnable.run(); } } + @Override + public void operationSucceeded() { + runInScope( + () -> { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationSucceeded(); + } + }); + } + @Override public void operationCancelled() { - @Nullable Scope scope = null; - try { - scope = inScope(); - } catch (RuntimeException e) { - // Ignore to prevent disrupting the lifecycle notification - } - try (Scope s = scope) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationCancelled(); - } - } + runInScope( + () -> { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationCancelled(); + } + }); } @Override public void operationFailed(Throwable error) { - @Nullable Scope scope = null; - try { - scope = inScope(); - } catch (RuntimeException e) { - // Ignore to prevent disrupting the lifecycle notification - } - try (Scope s = scope) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationFailed(error); - } - } + runInScope( + () -> { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationFailed(error); + } + }); } @Override From ec8f9005ea814c0c20312bf3fba6ae70f4b91b24 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:47:10 -0400 Subject: [PATCH 27/58] refactor(tracing): use NO_OP_SCOPE and enterScope in CompositeTracer --- .../api/gax/tracing/CompositeTracer.java | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 46c32e454912..6c1192e96af1 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Map; import org.jspecify.annotations.NullMarked; -import org.jspecify.annotations.Nullable; /** * A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of @@ -93,46 +92,51 @@ public Scope inScope() { }; } - private void runInScope(Runnable runnable) { - @Nullable Scope scope = null; + private static final Scope NO_OP_SCOPE = + new Scope() { + @Override + public void close() {} + }; + + /** + * Enters the tracer's ambient scope safely, returning a no-op {@link Scope} if entering fails. + * This avoids allocating runnables, throwing exceptions, or returning null during lifecycle + * notifications. + */ + private Scope enterScope() { try { - scope = inScope(); + return inScope(); } catch (RuntimeException e) { // Ignore to prevent disrupting the lifecycle notification - } - try (Scope s = scope) { - runnable.run(); + return NO_OP_SCOPE; } } @Override public void operationSucceeded() { - runInScope( - () -> { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationSucceeded(); - } - }); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationSucceeded(); + } + } } @Override public void operationCancelled() { - runInScope( - () -> { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationCancelled(); - } - }); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationCancelled(); + } + } } @Override public void operationFailed(Throwable error) { - runInScope( - () -> { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationFailed(error); - } - }); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationFailed(error); + } + } } @Override From d7721124d2e651d69e2ad52a2746335e448782e3 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:58:13 -0400 Subject: [PATCH 28/58] feat(tracing): scope attempt lifecycle callbacks in CompositeTracer --- .../api/gax/tracing/CompositeTracer.java | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 6c1192e96af1..2713b288b8b0 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -163,43 +163,55 @@ public void attemptStarted(Object request, int attemptNumber) { @Override public void attemptSucceeded() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptSucceeded(); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptSucceeded(); + } } } @Override public void attemptCancelled() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptCancelled(); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptCancelled(); + } } } @Override public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailed(error, delay); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailed(error, delay); + } } } @Override public void attemptFailedDuration(Throwable error, java.time.Duration delay) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailedDuration(error, delay); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailedDuration(error, delay); + } } } @Override public void attemptFailedRetriesExhausted(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailedRetriesExhausted(error); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailedRetriesExhausted(error); + } } } @Override public void attemptPermanentFailure(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptPermanentFailure(error); + try (Scope s = enterScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptPermanentFailure(error); + } } } From c9dacd50620108e2d387d66c900f6759bbbb3fc4 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:03:45 -0400 Subject: [PATCH 29/58] style(tracing): declare NO_OP_SCOPE at top of CompositeTracer --- .../com/google/api/gax/tracing/CompositeTracer.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 2713b288b8b0..f8bd92b6ad58 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -45,6 +45,12 @@ @NullMarked @InternalApi class CompositeTracer extends BaseApiTracer { + private static final Scope NO_OP_SCOPE = + new Scope() { + @Override + public void close() {} + }; + private final List children; public CompositeTracer(List children) { @@ -92,12 +98,6 @@ public Scope inScope() { }; } - private static final Scope NO_OP_SCOPE = - new Scope() { - @Override - public void close() {} - }; - /** * Enters the tracer's ambient scope safely, returning a no-op {@link Scope} if entering fails. * This avoids allocating runnables, throwing exceptions, or returning null during lifecycle From 9bbfc028c2ce7984fb34f52b1403480254f32b49 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:15:47 -0400 Subject: [PATCH 30/58] perf(tracing): optimize inScope fast path for 0 and 1 child tracers in CompositeTracer --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index f8bd92b6ad58..4d712c377792 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -59,6 +59,14 @@ public CompositeTracer(List children) { @Override public Scope inScope() { + if (children.isEmpty()) { + return NO_OP_SCOPE; + } + if (children.size() == 1) { + Scope scope = children.get(0).inScope(); + return scope != null ? scope : NO_OP_SCOPE; + } + final List childScopes = new ArrayList<>(children.size()); try { From 4d2d59271268622c9b17b49edd2ee65cf5076e3b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:18:12 -0400 Subject: [PATCH 31/58] perf(tracing): short-circuit enterScope when children is empty in CompositeTracer --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 4d712c377792..27ed5529398d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -112,6 +112,9 @@ public Scope inScope() { * notifications. */ private Scope enterScope() { + if (children.isEmpty()) { + return NO_OP_SCOPE; + } try { return inScope(); } catch (RuntimeException e) { From c8aa9311d0d2e991c2e421e05cb4131bd45eaf0b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:21:02 -0400 Subject: [PATCH 32/58] fix(tracing): ensure exception-safety across all throwables in inScope --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 27ed5529398d..81b770cee05c 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -76,11 +76,11 @@ public Scope inScope() { childScopes.add(scope); } } - } catch (RuntimeException e) { + } catch (RuntimeException | Error e) { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException suppressed) { + } catch (RuntimeException | Error suppressed) { e.addSuppressed(suppressed); } } @@ -92,9 +92,10 @@ public Scope inScope() { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException e) { + } catch (RuntimeException | Error e) { if (exception == null) { - exception = e; + exception = + e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); } else { exception.addSuppressed(e); } From c807e00d4de6bd5d461fb748d4fc30c1c009ba7f Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:23:24 -0400 Subject: [PATCH 33/58] perf(tracing): avoid lambda allocation when childScopes is empty or singleton in inScope --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 81b770cee05c..c9f0fd3d9662 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -76,6 +76,12 @@ public Scope inScope() { childScopes.add(scope); } } + if (childScopes.isEmpty()) { + return NO_OP_SCOPE; + } + if (childScopes.size() == 1) { + return childScopes.get(0); + } } catch (RuntimeException | Error e) { for (int i = childScopes.size() - 1; i >= 0; i--) { try { @@ -113,9 +119,6 @@ public Scope inScope() { * notifications. */ private Scope enterScope() { - if (children.isEmpty()) { - return NO_OP_SCOPE; - } try { return inScope(); } catch (RuntimeException e) { From ea41ebddb8e0928f38aca78e62128e85a1721868 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:26:01 -0400 Subject: [PATCH 34/58] fix(tracing): catch RuntimeException | Error in enterScope --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index c9f0fd3d9662..3e12df90f00a 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -121,7 +121,7 @@ public Scope inScope() { private Scope enterScope() { try { return inScope(); - } catch (RuntimeException e) { + } catch (RuntimeException | Error e) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From dd2c8f71f565b3ec8268c439acd6b1b37c630caa Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:27:08 -0400 Subject: [PATCH 35/58] fix(tracing): preserve Error type without wrapping in RuntimeException in CompositeTracer --- .../com/google/api/gax/tracing/CompositeTracer.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 3e12df90f00a..a25c877661b2 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -94,21 +94,24 @@ public Scope inScope() { } return () -> { - RuntimeException exception = null; + Throwable exception = null; for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); } catch (RuntimeException | Error e) { if (exception == null) { - exception = - e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); + exception = e; } else { exception.addSuppressed(e); } } } if (exception != null) { - throw exception; + if (exception instanceof RuntimeException) { + throw (RuntimeException) exception; + } else { + throw (Error) exception; + } } }; } From 779a1aee831d73aed0958dfff4ef74fcb74bb9f1 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:29:30 -0400 Subject: [PATCH 36/58] fix(tracing): catch Throwable in close loop and restrict enterScope to RuntimeException --- .../com/google/api/gax/tracing/CompositeTracer.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index a25c877661b2..ae20eacad6a0 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -98,19 +98,21 @@ public Scope inScope() { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException | Error e) { + } catch (Throwable t) { if (exception == null) { - exception = e; + exception = t; } else { - exception.addSuppressed(e); + exception.addSuppressed(t); } } } if (exception != null) { if (exception instanceof RuntimeException) { throw (RuntimeException) exception; - } else { + } else if (exception instanceof Error) { throw (Error) exception; + } else { + throw new RuntimeException(exception); } } }; @@ -124,7 +126,7 @@ public Scope inScope() { private Scope enterScope() { try { return inScope(); - } catch (RuntimeException | Error e) { + } catch (RuntimeException e) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From cac56404191d8ffce65bb7b74a2418e6d8eacd85 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:33:28 -0400 Subject: [PATCH 37/58] fix(tracing): catch RuntimeException | Error in childScopes close and simplify rethrow --- .../com/google/api/gax/tracing/CompositeTracer.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index ae20eacad6a0..57dabc9aa0ee 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -98,21 +98,19 @@ public Scope inScope() { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (Throwable t) { + } catch (RuntimeException | Error e) { if (exception == null) { - exception = t; + exception = e; } else { - exception.addSuppressed(t); + exception.addSuppressed(e); } } } if (exception != null) { if (exception instanceof RuntimeException) { throw (RuntimeException) exception; - } else if (exception instanceof Error) { - throw (Error) exception; } else { - throw new RuntimeException(exception); + throw (Error) exception; } } }; From c4055816e8f61c657ba91537628add3c26ccdfdc Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:50:48 -0400 Subject: [PATCH 38/58] refactor(tracing): extract CompositeScope class in CompositeTracer --- .../api/gax/tracing/CompositeTracer.java | 55 ++++++++++++------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 57dabc9aa0ee..68c03df378e5 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -82,38 +82,55 @@ public Scope inScope() { if (childScopes.size() == 1) { return childScopes.get(0); } - } catch (RuntimeException | Error e) { + return new CompositeScope(childScopes); + } catch (Throwable t) { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException | Error suppressed) { - e.addSuppressed(suppressed); + } catch (Throwable suppressed) { + t.addSuppressed(suppressed); } } - throw e; + throwException(t); + return NO_OP_SCOPE; } + } - return () -> { - Throwable exception = null; - for (int i = childScopes.size() - 1; i >= 0; i--) { + private static class CompositeScope implements Scope { + private final List scopes; + + CompositeScope(List scopes) { + this.scopes = scopes; + } + + @Override + public void close() { + Throwable firstException = null; + for (int i = scopes.size() - 1; i >= 0; i--) { try { - childScopes.get(i).close(); - } catch (RuntimeException | Error e) { - if (exception == null) { - exception = e; + scopes.get(i).close(); + } catch (Throwable t) { + if (firstException == null) { + firstException = t; } else { - exception.addSuppressed(e); + firstException.addSuppressed(t); } } } - if (exception != null) { - if (exception instanceof RuntimeException) { - throw (RuntimeException) exception; - } else { - throw (Error) exception; - } + if (firstException != null) { + throwException(firstException); } - }; + } + } + + private static void throwException(Throwable t) { + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } else if (t instanceof Error) { + throw (Error) t; + } else { + throw new RuntimeException(t); + } } /** From 64e5795c9bf03c38b338bc8c6aaf3e6e331191b9 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:52:53 -0400 Subject: [PATCH 39/58] fix(tracing): catch RuntimeException | Error in enterScope to prevent lifecycle disruption --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 68c03df378e5..a74d5e0cf917 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -141,7 +141,7 @@ private static void throwException(Throwable t) { private Scope enterScope() { try { return inScope(); - } catch (RuntimeException e) { + } catch (RuntimeException | Error e) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From 7a0a6711f0238619c42c48345adc1bfbf21bd6bd Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:55:34 -0400 Subject: [PATCH 40/58] fix(tracing): throw throwException in CompositeTracer and catch Throwable in enterScope --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index a74d5e0cf917..d1fe0e3bef08 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -91,8 +91,7 @@ public Scope inScope() { t.addSuppressed(suppressed); } } - throwException(t); - return NO_OP_SCOPE; + throw throwException(t); } } @@ -118,12 +117,12 @@ public void close() { } } if (firstException != null) { - throwException(firstException); + throw throwException(firstException); } } } - private static void throwException(Throwable t) { + private static RuntimeException throwException(Throwable t) { if (t instanceof RuntimeException) { throw (RuntimeException) t; } else if (t instanceof Error) { @@ -141,7 +140,7 @@ private static void throwException(Throwable t) { private Scope enterScope() { try { return inScope(); - } catch (RuntimeException | Error e) { + } catch (Throwable t) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From f1647d25ec27eed3e9b91145e60f38d83e216ed8 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:57:58 -0400 Subject: [PATCH 41/58] refactor(tracing): simplify NO_OP_SCOPE lambda and catch RuntimeException in enterScope --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index d1fe0e3bef08..a78d6f3ddbd9 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -45,11 +45,7 @@ @NullMarked @InternalApi class CompositeTracer extends BaseApiTracer { - private static final Scope NO_OP_SCOPE = - new Scope() { - @Override - public void close() {} - }; + private static final Scope NO_OP_SCOPE = () -> {}; private final List children; @@ -140,7 +136,7 @@ private static RuntimeException throwException(Throwable t) { private Scope enterScope() { try { return inScope(); - } catch (Throwable t) { + } catch (RuntimeException e) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From 5e1d1183c807b1701ef46ef1410b551d96e55a3c Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:00:11 -0400 Subject: [PATCH 42/58] refactor(tracing): return RuntimeException from throwException in CompositeTracer --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index a78d6f3ddbd9..f28fbcfdf3c6 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -120,11 +120,11 @@ public void close() { private static RuntimeException throwException(Throwable t) { if (t instanceof RuntimeException) { - throw (RuntimeException) t; + return (RuntimeException) t; } else if (t instanceof Error) { throw (Error) t; } else { - throw new RuntimeException(t); + return new RuntimeException(t); } } From e83556542114438575fdb7e347533f362d5020ed Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:10:11 -0400 Subject: [PATCH 43/58] refactor(tracing): optimize CompositeTracer scope allocation and exception handling --- .../api/gax/tracing/CompositeTracer.java | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index f28fbcfdf3c6..baf5417bd4d5 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -31,7 +31,6 @@ import com.google.api.core.InternalApi; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; import java.util.Map; import org.jspecify.annotations.NullMarked; @@ -63,26 +62,27 @@ public Scope inScope() { return scope != null ? scope : NO_OP_SCOPE; } - final List childScopes = new ArrayList<>(children.size()); + Scope[] childScopes = new Scope[children.size()]; + int scopeCount = 0; try { for (ApiTracer child : children) { Scope scope = child.inScope(); if (scope != null) { - childScopes.add(scope); + childScopes[scopeCount++] = scope; } } - if (childScopes.isEmpty()) { + if (scopeCount == 0) { return NO_OP_SCOPE; } - if (childScopes.size() == 1) { - return childScopes.get(0); + if (scopeCount == 1) { + return childScopes[0]; } - return new CompositeScope(childScopes); + return new CompositeScope(childScopes, scopeCount); } catch (Throwable t) { - for (int i = childScopes.size() - 1; i >= 0; i--) { + for (int i = scopeCount - 1; i >= 0; i--) { try { - childScopes.get(i).close(); + childScopes[i].close(); } catch (Throwable suppressed) { t.addSuppressed(suppressed); } @@ -92,18 +92,20 @@ public Scope inScope() { } private static class CompositeScope implements Scope { - private final List scopes; + private final Scope[] scopes; + private final int count; - CompositeScope(List scopes) { + CompositeScope(Scope[] scopes, int count) { this.scopes = scopes; + this.count = count; } @Override public void close() { Throwable firstException = null; - for (int i = scopes.size() - 1; i >= 0; i--) { + for (int i = count - 1; i >= 0; i--) { try { - scopes.get(i).close(); + scopes[i].close(); } catch (Throwable t) { if (firstException == null) { firstException = t; @@ -121,11 +123,11 @@ public void close() { private static RuntimeException throwException(Throwable t) { if (t instanceof RuntimeException) { return (RuntimeException) t; - } else if (t instanceof Error) { + } + if (t instanceof Error) { throw (Error) t; - } else { - return new RuntimeException(t); } + return new RuntimeException(t); } /** @@ -136,7 +138,7 @@ private static RuntimeException throwException(Throwable t) { private Scope enterScope() { try { return inScope(); - } catch (RuntimeException e) { + } catch (Throwable t) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From db7203f5d5f3229e466f86986783f572d9beca79 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:30:18 -0400 Subject: [PATCH 44/58] fix(tracing): guard against self-suppression in CompositeTracer --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index baf5417bd4d5..7ffb879a01ed 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -84,7 +84,9 @@ public Scope inScope() { try { childScopes[i].close(); } catch (Throwable suppressed) { - t.addSuppressed(suppressed); + if (t != suppressed) { + t.addSuppressed(suppressed); + } } } throw throwException(t); @@ -109,7 +111,7 @@ public void close() { } catch (Throwable t) { if (firstException == null) { firstException = t; - } else { + } else if (firstException != t) { firstException.addSuppressed(t); } } From 0c20afd6b46d82c8dae963f8699551accb5c478c Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:40:14 -0400 Subject: [PATCH 45/58] fix(tracing): catch RuntimeException in CompositeTracer enterScope --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 7ffb879a01ed..e8cdf2e1b500 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -140,7 +140,7 @@ private static RuntimeException throwException(Throwable t) { private Scope enterScope() { try { return inScope(); - } catch (Throwable t) { + } catch (RuntimeException e) { // Ignore to prevent disrupting the lifecycle notification return NO_OP_SCOPE; } From 5a3fb6b49295c07ecef640e5322068e1f0dec934 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:29:15 -0400 Subject: [PATCH 46/58] docs(tracing): add Javadoc and explanatory comments to CompositeTracer and CompositeScope --- .../api/gax/tracing/CompositeTracer.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index e8cdf2e1b500..6b89e5781a0e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -93,15 +93,25 @@ public Scope inScope() { } } + /** + * An aggregate {@link Scope} that encapsulates and closes multiple child scopes in reverse order. + */ private static class CompositeScope implements Scope { private final Scope[] scopes; private final int count; + /** + * Constructs a {@code CompositeScope} managing the given array of active child scopes. + * + * @param scopes the array containing active child scopes + * @param count the number of valid scopes in the array + */ CompositeScope(Scope[] scopes, int count) { this.scopes = scopes; this.count = count; } + /** Closes all managed child scopes in reverse order, suppressing secondary exceptions. */ @Override public void close() { Throwable firstException = null; @@ -122,6 +132,12 @@ public void close() { } } + /** + * Rethrows or wraps a {@link Throwable} without losing runtime exception or error fidelity. + * + * @param t the throwable to rethrow or wrap + * @return a {@link RuntimeException} wrapping {@code t} if {@code t} is a checked exception + */ private static RuntimeException throwException(Throwable t) { if (t instanceof RuntimeException) { return (RuntimeException) t; @@ -136,6 +152,10 @@ private static RuntimeException throwException(Throwable t) { * Enters the tracer's ambient scope safely, returning a no-op {@link Scope} if entering fails. * This avoids allocating runnables, throwing exceptions, or returning null during lifecycle * notifications. + * + *

Entering the ambient scope ensures active trace span context is present on the thread so + * that OpenTelemetry metric measurements recorded in lifecycle callbacks attach exemplars + * pointing to the active span. */ private Scope enterScope() { try { From 55cee78cca173af07409440687fd0b43295bb235 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:38:57 -0400 Subject: [PATCH 47/58] fix(tracing): make CompositeScope.close idempotent and clear references --- .../api/gax/tracing/CompositeTracer.java | 18 +++++++++------- .../api/gax/tracing/CompositeTracerTest.java | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 6b89e5781a0e..e3c8461ae859 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -116,13 +116,17 @@ private static class CompositeScope implements Scope { public void close() { Throwable firstException = null; for (int i = count - 1; i >= 0; i--) { - try { - scopes[i].close(); - } catch (Throwable t) { - if (firstException == null) { - firstException = t; - } else if (firstException != t) { - firstException.addSuppressed(t); + Scope scope = scopes[i]; + if (scope != null) { + scopes[i] = null; + try { + scope.close(); + } catch (Throwable t) { + if (firstException == null) { + firstException = t; + } else if (firstException != t) { + firstException.addSuppressed(t); + } } } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index 02e5012dc32c..37881959c3e4 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -125,6 +126,26 @@ void testInScope_childScopeCloseThrows() { inOrder.verify(scope1).close(); } + @Test + void testInScope_compositeScopeClose_isIdempotent() { + ApiTracer.Scope scope1 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + ApiTracer.Scope scope2 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + + when(child1.inScope()).thenReturn(scope1); + when(child2.inScope()).thenReturn(scope2); + + ApiTracer.Scope compositeScope = compositeTracer.inScope(); + + compositeScope.close(); + // Subsequent close should be idempotent and not invoke underlying scopes again + compositeScope.close(); + + verify(scope2, times(1)).close(); + verify(scope1, times(1)).close(); + } + @Test void testOperationSucceeded() { compositeTracer.operationSucceeded(); From 92a9458e13b389f8bda8873148e3ec2b2aa00539 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 12:12:08 -0400 Subject: [PATCH 48/58] test(showcase): add GAPIC Showcase integration tests for OpenTelemetry T3 tracing --- .../v1beta1/it/ITOtelT3MetricsExemplar.java | 264 +++++++++ .../v1beta1/it/ITOtelT3T4Hierarchy.java | 547 ++++++++++++++++++ .../showcase/v1beta1/it/ITOtelT3Tracing.java | 492 ++++++++++++++++ 3 files changed, 1303 insertions(+) create mode 100644 java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java create mode 100644 java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java create mode 100644 java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java new file mode 100644 index 000000000000..5aa972570b22 --- /dev/null +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -0,0 +1,264 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.showcase.v1beta1.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.CompositeTracerFactory; +import com.google.api.gax.tracing.OpenTelemetryMetricsFactory; +import com.google.api.gax.tracing.OpenTelemetryTracingFactory; +import com.google.showcase.v1beta1.EchoClient; +import com.google.showcase.v1beta1.EchoRequest; +import com.google.showcase.v1beta1.EchoSettings; +import com.google.showcase.v1beta1.stub.EchoStub; +import com.google.showcase.v1beta1.stub.EchoStubSettings; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.ExemplarFilter; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.ExemplarData; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Integration tests for Feature 3 (F3.1–F3.2): T3 Tracing and M3 Metrics Exemplar Correlation. */ +class ITOtelT3MetricsExemplar { + private static final String SHOWCASE_SERVER_ADDRESS = "localhost"; + private static final long SHOWCASE_SERVER_PORT = 7469; + private static final String SHOWCASE_GRPC_ENDPOINT = + String.format("%s:%s", SHOWCASE_SERVER_ADDRESS, SHOWCASE_SERVER_PORT); + private static final String SHOWCASE_HTTPJSON_ENDPOINT = + String.format("http://%s:%s", SHOWCASE_SERVER_ADDRESS, SHOWCASE_SERVER_PORT); + private static final String SHOWCASE_SERVICE_NAME = "showcase"; + + private InMemorySpanExporter spanExporter; + private InMemoryMetricReader metricReader; + private OpenTelemetrySdk openTelemetrySdk; + + @BeforeEach + void setUp() { + spanExporter = InMemorySpanExporter.create(); + metricReader = InMemoryMetricReader.create(); + + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + + SdkMeterProvider meterProvider = + SdkMeterProvider.builder() + .registerMetricReader(metricReader) + .setExemplarFilter(ExemplarFilter.traceBased()) + .build(); + + openTelemetrySdk = + OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setMeterProvider(meterProvider) + .buildAndRegisterGlobal(); + } + + @AfterEach + void tearDown() { + if (openTelemetrySdk != null) { + openTelemetrySdk.close(); + } + GlobalOpenTelemetry.resetForTest(); + } + + // F3.1: HTTP M3 metric records T3 span as exemplar + @Test + void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { + ApiTracerFactory compositeTracerFactory = createCompositeTracerFactory(); + EchoSettings settings = createEchoSettings(true); + EchoStub stub = createStubWithServiceName(settings, compositeTracerFactory); + + try (EchoClient client = EchoClient.create(stub)) { + client.echo(EchoRequest.newBuilder().setContent("exemplar-test-http").build()); + + List spans = waitAndCollectSpans(2); + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + Collection metrics = waitAndCollectMetrics(); + MetricData durationMetric = + metrics.stream() + .filter(m -> m.getName().equals("gcp.client.request.duration")) + .findFirst() + .orElseThrow(() -> new AssertionError("Duration metric not found in: " + metrics)); + + HistogramPointData point = durationMetric.getHistogramData().getPoints().iterator().next(); + List exemplars = new ArrayList<>(point.getExemplars()); + assertThat(exemplars).isNotEmpty(); + + ExemplarData exemplar = exemplars.get(0); + assertThat(exemplar.getSpanContext().getTraceId()).isEqualTo(t3Span.getTraceId()); + assertThat(exemplar.getSpanContext().getSpanId()).isEqualTo(t3Span.getSpanId()); + } + } + + // F3.2: gRPC M3 metric records T3 span as exemplar + @Test + void testGrpc_m3ExemplarMatchesT3Span() throws Exception { + ApiTracerFactory compositeTracerFactory = createCompositeTracerFactory(); + EchoSettings settings = createEchoSettings(false); + EchoStub stub = createStubWithServiceName(settings, compositeTracerFactory); + + try (EchoClient client = EchoClient.create(stub)) { + client.echo(EchoRequest.newBuilder().setContent("exemplar-test-grpc").build()); + + List spans = waitAndCollectSpans(2); + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + Collection metrics = waitAndCollectMetrics(); + MetricData durationMetric = + metrics.stream() + .filter(m -> m.getName().equals("gcp.client.request.duration")) + .findFirst() + .orElseThrow(() -> new AssertionError("Duration metric not found in: " + metrics)); + + HistogramPointData point = durationMetric.getHistogramData().getPoints().iterator().next(); + List exemplars = new ArrayList<>(point.getExemplars()); + assertThat(exemplars).isNotEmpty(); + + ExemplarData exemplar = exemplars.get(0); + assertThat(exemplar.getSpanContext().getTraceId()).isEqualTo(t3Span.getTraceId()); + assertThat(exemplar.getSpanContext().getSpanId()).isEqualTo(t3Span.getSpanId()); + } + } + + private CompositeTracerFactory createCompositeTracerFactory() { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + OpenTelemetryMetricsFactory metricsFactory = new OpenTelemetryMetricsFactory(openTelemetrySdk); + return new CompositeTracerFactory(Arrays.asList(tracingFactory, metricsFactory)); + } + + private List waitAndCollectSpans(int minSpans) { + Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); + return spanExporter.getFinishedSpanItems(); + } + + private Collection waitAndCollectMetrics() { + java.util.concurrent.atomic.AtomicReference> holder = + new java.util.concurrent.atomic.AtomicReference<>(); + Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .until( + () -> { + Collection metrics = metricReader.collectAllMetrics(); + if (metrics.isEmpty()) { + return false; + } + for (MetricData md : metrics) { + if (md.getName().equals("gcp.client.request.duration")) { + for (HistogramPointData p : md.getHistogramData().getPoints()) { + if (!p.getExemplars().isEmpty()) { + holder.set(metrics); + return true; + } + } + } + } + return false; + }); + return holder.get(); + } + + private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { + if (isHttpJson) { + return EchoSettings.newHttpJsonBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + EchoSettings.defaultHttpJsonTransportProviderBuilder() + .setHttpTransport( + new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .build()) + .setEndpoint(SHOWCASE_HTTPJSON_ENDPOINT) + .build(); + } else { + return EchoSettings.newBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + EchoSettings.defaultGrpcTransportProviderBuilder() + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .build()) + .setEndpoint(SHOWCASE_GRPC_ENDPOINT) + .build(); + } + } + + private EchoStub createStubWithServiceName(EchoSettings settings, ApiTracerFactory tracerFactory) + throws IOException { + EchoStubSettings.Builder builder = + (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); + builder.setTracerFactory(tracerFactory); + return new ExtendedEchoStubSettings(builder).createStub(); + } + + private static class ExtendedEchoStubSettings extends EchoStubSettings { + protected ExtendedEchoStubSettings(EchoStubSettings.Builder builder) throws IOException { + super(builder); + } + + @Override + public String getServiceName() { + return SHOWCASE_SERVICE_NAME; + } + } +} diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java new file mode 100644 index 000000000000..23714a67aef0 --- /dev/null +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java @@ -0,0 +1,547 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.showcase.v1beta1.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.StatusCode; +import com.google.api.gax.tracing.ObservabilityAttributes; +import com.google.api.gax.tracing.OpenTelemetryTracingFactory; +import com.google.common.collect.ImmutableSet; +import com.google.rpc.Code; +import com.google.rpc.Status; +import com.google.showcase.v1beta1.AttemptSequenceRequest; +import com.google.showcase.v1beta1.CreateSequenceRequest; +import com.google.showcase.v1beta1.Sequence; +import com.google.showcase.v1beta1.SequenceServiceClient; +import com.google.showcase.v1beta1.SequenceServiceSettings; +import com.google.showcase.v1beta1.it.util.TestClientInitializer; +import com.google.showcase.v1beta1.stub.SequenceServiceStubSettings; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.io.IOException; +import java.time.Duration; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Integration tests for Feature 2 (F2.1–F2.4): T3/T4 Span Hierarchy and Retry Aggregation. */ +class ITOtelT3T4Hierarchy { + private static final String SHOWCASE_SERVER_ADDRESS = "localhost"; + private static final long SHOWCASE_SERVER_PORT = 7469; + private static final String SHOWCASE_GRPC_ENDPOINT = + String.format("%s:%s", SHOWCASE_SERVER_ADDRESS, SHOWCASE_SERVER_PORT); + private static final String SHOWCASE_HTTPJSON_ENDPOINT = + String.format("http://%s:%s", SHOWCASE_SERVER_ADDRESS, SHOWCASE_SERVER_PORT); + private static final String SHOWCASE_SERVICE_NAME = "showcase"; + + private InMemorySpanExporter spanExporter; + private OpenTelemetrySdk openTelemetrySdk; + private SequenceServiceClient setupGrpcClient; + private SequenceServiceClient setupHttpJsonClient; + + @BeforeEach + void setUp() throws Exception { + spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + openTelemetrySdk = + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + + setupGrpcClient = TestClientInitializer.createGrpcSequenceClient(); + setupHttpJsonClient = TestClientInitializer.createHttpJsonSequenceClient(); + } + + @AfterEach + void tearDown() { + if (setupGrpcClient != null) { + setupGrpcClient.close(); + } + if (setupHttpJsonClient != null) { + setupHttpJsonClient.close(); + } + if (openTelemetrySdk != null) { + openTelemetrySdk.close(); + } + GlobalOpenTelemetry.resetForTest(); + } + + // F2.1: HTTP T3/T4 retry succeeds (1 T3 span, 2 T4 child spans) + @Test + void testHttpJson_retrySucceeds() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + + // Sequence: attempt 1 -> UNAVAILABLE, attempt 2 -> OK + Sequence sequence = + Sequence.newBuilder() + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.OK.getNumber()).build()) + .build()) + .build(); + Sequence createdSequence = + setupHttpJsonClient.createSequence( + CreateSequenceRequest.newBuilder().setSequence(sequence).build()); + + // Flush spans created by setup client call + spanExporter.reset(); + + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofMillis(50L)) + .setRetryDelayMultiplier(1.5) + .setMaxRetryDelayDuration(Duration.ofMillis(200L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setTotalTimeoutDuration(Duration.ofMillis(3000L)) + .setMaxAttempts(3) + .build(); + + SequenceServiceClient client = + createSequenceClient( + true, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); + + try { + client.attemptSequence( + AttemptSequenceRequest.newBuilder().setName(createdSequence.getName()).build()); + + List spans = waitAndCollectSpans(3); + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + List t4Spans = + spans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .sorted(Comparator.comparingLong(SpanData::getStartEpochNanos)) + .collect(Collectors.toList()); + + assertThat(t4Spans).hasSize(2); + + // Verify T3/T4 Parentage: both T4 child spans have parent_span_id == T3.span_id + for (SpanData t4 : t4Spans) { + assertThat(t4.getParentSpanId()).isEqualTo(t3Span.getSpanId()); + } + + // Verify T3 retry aggregation on success + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNull(); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("OK"); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo(200L); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.HTTP_URL_TEMPLATE_ATTRIBUTE))) + .isNotEmpty(); + } finally { + client.close(); + } + } + + // F2.2: HTTP T3/T4 retries exhausted + @Test + void testHttpJson_retriesExhausted() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + + // Sequence: 3 UNAVAILABLE responses + Sequence sequence = + Sequence.newBuilder() + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .build(); + Sequence createdSequence = + setupHttpJsonClient.createSequence( + CreateSequenceRequest.newBuilder().setSequence(sequence).build()); + + spanExporter.reset(); + + // maxAttempts = 2 will exhaust retries after 2 attempts + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofMillis(50L)) + .setRetryDelayMultiplier(1.5) + .setMaxRetryDelayDuration(Duration.ofMillis(200L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setTotalTimeoutDuration(Duration.ofMillis(3000L)) + .setMaxAttempts(2) + .build(); + + SequenceServiceClient client = + createSequenceClient( + true, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); + + try { + assertThrows( + ApiException.class, + () -> + client.attemptSequence( + AttemptSequenceRequest.newBuilder().setName(createdSequence.getName()).build())); + + List spans = waitAndCollectSpans(3); + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + List t4Spans = + spans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .sorted(Comparator.comparingLong(SpanData::getStartEpochNanos)) + .collect(Collectors.toList()); + + assertThat(t4Spans).hasSize(2); + + // Verify T3/T4 Parentage: all T4 child spans have parent_span_id == T3.span_id + for (SpanData t4 : t4Spans) { + assertThat(t4.getParentSpanId()).isEqualTo(t3Span.getSpanId()); + } + + // Verify T3 attributes on retries exhausted + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo(503L); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("UNAVAILABLE"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isNotEmpty(); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } finally { + client.close(); + } + } + + // F2.3: gRPC T3/T4 retry succeeds + @Test + void testGrpc_retrySucceeds() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + + // Sequence: attempt 1 -> UNAVAILABLE, attempt 2 -> OK + Sequence sequence = + Sequence.newBuilder() + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.OK.getNumber()).build()) + .build()) + .build(); + Sequence createdSequence = + setupGrpcClient.createSequence( + CreateSequenceRequest.newBuilder().setSequence(sequence).build()); + + spanExporter.reset(); + + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofMillis(50L)) + .setRetryDelayMultiplier(1.5) + .setMaxRetryDelayDuration(Duration.ofMillis(200L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setTotalTimeoutDuration(Duration.ofMillis(3000L)) + .setMaxAttempts(3) + .build(); + + SequenceServiceClient client = + createSequenceClient( + false, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); + + try { + client.attemptSequence( + AttemptSequenceRequest.newBuilder().setName(createdSequence.getName()).build()); + + List spans = waitAndCollectSpans(3); + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + List t4Spans = + spans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .sorted(Comparator.comparingLong(SpanData::getStartEpochNanos)) + .collect(Collectors.toList()); + + assertThat(t4Spans).hasSize(2); + + // Verify T3/T4 Parentage: both T4 child spans have parent_span_id == T3.span_id + for (SpanData t4 : t4Spans) { + assertThat(t4.getParentSpanId()).isEqualTo(t3Span.getSpanId()); + } + + // Verify T3 retry aggregation on success + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNull(); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("OK"); + } finally { + client.close(); + } + } + + // F2.4: gRPC T3/T4 retries exhausted + @Test + void testGrpc_retriesExhausted() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + + // Sequence: 3 UNAVAILABLE responses + Sequence sequence = + Sequence.newBuilder() + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .addResponses( + Sequence.Response.newBuilder() + .setStatus(Status.newBuilder().setCode(Code.UNAVAILABLE.getNumber()).build()) + .build()) + .build(); + Sequence createdSequence = + setupGrpcClient.createSequence( + CreateSequenceRequest.newBuilder().setSequence(sequence).build()); + + spanExporter.reset(); + + // maxAttempts = 2 will exhaust retries after 2 attempts + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofMillis(50L)) + .setRetryDelayMultiplier(1.5) + .setMaxRetryDelayDuration(Duration.ofMillis(200L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(1000L)) + .setTotalTimeoutDuration(Duration.ofMillis(3000L)) + .setMaxAttempts(2) + .build(); + + SequenceServiceClient client = + createSequenceClient( + false, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); + + try { + assertThrows( + ApiException.class, + () -> + client.attemptSequence( + AttemptSequenceRequest.newBuilder().setName(createdSequence.getName()).build())); + + List spans = waitAndCollectSpans(3); + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + List t4Spans = + spans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .sorted(Comparator.comparingLong(SpanData::getStartEpochNanos)) + .collect(Collectors.toList()); + + assertThat(t4Spans).hasSize(2); + + // Verify T3/T4 Parentage: all T4 child spans have parent_span_id == T3.span_id + for (SpanData t4 : t4Spans) { + assertThat(t4.getParentSpanId()).isEqualTo(t3Span.getSpanId()); + } + + // Verify T3 attributes on retries exhausted + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("UNAVAILABLE"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isNotEmpty(); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } finally { + client.close(); + } + } + + private List waitAndCollectSpans(int minSpans) { + org.awaitility.Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); + return spanExporter.getFinishedSpanItems(); + } + + private SequenceServiceClient createSequenceClient( + boolean isHttpJson, + OpenTelemetryTracingFactory tracingFactory, + RetrySettings retrySettings, + Set retryableCodes) + throws Exception { + SequenceServiceSettings.Builder settingsBuilder = + isHttpJson + ? SequenceServiceSettings.newHttpJsonBuilder() + : SequenceServiceSettings.newBuilder(); + + settingsBuilder + .attemptSequenceSettings() + .setRetrySettings(retrySettings) + .setRetryableCodes(retryableCodes); + + settingsBuilder.setCredentialsProvider(NoCredentialsProvider.create()); + + if (isHttpJson) { + settingsBuilder + .setTransportChannelProvider( + SequenceServiceSettings.defaultHttpJsonTransportProviderBuilder() + .setHttpTransport( + new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .build()) + .setEndpoint(SHOWCASE_HTTPJSON_ENDPOINT); + } else { + settingsBuilder + .setTransportChannelProvider( + SequenceServiceSettings.defaultGrpcTransportProviderBuilder() + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .build()) + .setEndpoint(SHOWCASE_GRPC_ENDPOINT); + } + + SequenceServiceStubSettings.Builder stubSettingsBuilder = + settingsBuilder.getStubSettingsBuilder(); + stubSettingsBuilder.setTracerFactory(tracingFactory); + return SequenceServiceClient.create( + new ExtendedSequenceServiceStubSettings(stubSettingsBuilder).createStub()); + } + + private static class ExtendedSequenceServiceStubSettings extends SequenceServiceStubSettings { + protected ExtendedSequenceServiceStubSettings(SequenceServiceStubSettings.Builder builder) + throws IOException { + super(builder); + } + + @Override + public String getServiceName() { + return SHOWCASE_SERVICE_NAME; + } + } +} diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java new file mode 100644 index 000000000000..3763b24f4212 --- /dev/null +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -0,0 +1,492 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.showcase.v1beta1.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.tracing.ObservabilityAttributes; +import com.google.api.gax.tracing.OpenTelemetryTracingFactory; +import com.google.rpc.Code; +import com.google.rpc.Status; +import com.google.showcase.v1beta1.BlockRequest; +import com.google.showcase.v1beta1.BlockResponse; +import com.google.showcase.v1beta1.EchoClient; +import com.google.showcase.v1beta1.EchoRequest; +import com.google.showcase.v1beta1.EchoSettings; +import com.google.showcase.v1beta1.stub.EchoStub; +import com.google.showcase.v1beta1.stub.EchoStubSettings; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.io.IOException; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.threeten.bp.Duration; + +/** Integration tests for Feature 1 (F1.1–F1.8): Client-level T3 tracing across HTTP and gRPC. */ +class ITOtelT3Tracing { + private static final String SHOWCASE_SERVER_ADDRESS = "localhost"; + private static final long SHOWCASE_SERVER_PORT = 7469; + private static final String SHOWCASE_GRPC_ENDPOINT = + String.format("%s:%s", SHOWCASE_SERVER_ADDRESS, SHOWCASE_SERVER_PORT); + private static final String SHOWCASE_HTTPJSON_ENDPOINT = + String.format("http://%s:%s", SHOWCASE_SERVER_ADDRESS, SHOWCASE_SERVER_PORT); + private static final String SHOWCASE_SERVICE_NAME = "showcase"; + + private InMemorySpanExporter spanExporter; + private OpenTelemetrySdk openTelemetrySdk; + + @BeforeEach + void setUp() { + spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + openTelemetrySdk = + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + } + + @AfterEach + void tearDown() { + if (openTelemetrySdk != null) { + openTelemetrySdk.close(); + } + GlobalOpenTelemetry.resetForTest(); + } + + // F1.1: HTTP no traces emitted unless enabled. + @Test + void testTracingDisabled_httpjson() throws Exception { + EchoSettings settings = createEchoSettings(true); + try (EchoClient client = EchoClient.create(settings)) { + client.echo(EchoRequest.newBuilder().setContent("test-f1-1").build()); + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).isEmpty(); + } + } + + // F1.2: HTTP T3 success case name and attributes conform to requirements. + @Test + void testT3Success_httpjson() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + EchoSettings settings = createEchoSettings(true); + EchoStub stub = createStubWithServiceName(settings, tracingFactory); + + try (EchoClient client = EchoClient.create(stub)) { + client.echo(EchoRequest.newBuilder().setContent("test-f1-2").build()); + + List spans = waitAndCollectSpans(2); + assertThat(spans).isNotEmpty(); + + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + assertThat(t3Span.getKind()).isEqualTo(SpanKind.INTERNAL); + assertThat(t3Span.getName()).isNotEmpty(); + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE))) + .isEqualTo("http"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.SERVER_ADDRESS_ATTRIBUTE))) + .isEqualTo(SHOWCASE_SERVER_ADDRESS); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.longKey(ObservabilityAttributes.SERVER_PORT_ATTRIBUTE))) + .isEqualTo(SHOWCASE_SERVER_PORT); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE))) + .isEqualTo(SHOWCASE_SERVICE_NAME); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo(200L); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.HTTP_URL_TEMPLATE_ATTRIBUTE))) + .isEqualTo("v1beta1/echo:echo"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNull(); + } + } + + // F1.3: HTTP T3 server failures case name and attributes conform to requirements. + @Test + void testT3ServerFailure_httpjson() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + EchoSettings settings = createEchoSettings(true); + EchoStub stub = createStubWithServiceName(settings, tracingFactory); + + try (EchoClient client = EchoClient.create(stub)) { + EchoRequest request = + EchoRequest.newBuilder() + .setError( + Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT.getNumber()) + .setMessage("Server-side failure message") + .build()) + .build(); + + assertThrows(ApiException.class, () -> client.echo(request)); + + List spans = waitAndCollectSpans(2); + assertThat(spans).isNotEmpty(); + + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))) + .isAtLeast(400L); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isNotEmpty(); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } + } + + // F1.4: HTTP T3 client failures case name and attributes conform to requirements. + @Test + void testT3ClientFailure_httpjson() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + EchoSettings settings = createEchoSettings(true); + // Configure 50ms timeout for blockCallable + EchoStubSettings.Builder builder = + (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); + builder.setTracerFactory(tracingFactory); + builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(50L)); + EchoStub stub = new ExtendedEchoStubSettings(builder).createStub(); + + try (EchoClient client = EchoClient.create(stub)) { + BlockRequest request = + BlockRequest.newBuilder() + .setSuccess(BlockResponse.newBuilder().setContent("content").build()) + .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(3).build()) + .build(); + + assertThrows(Exception.class, () -> client.block(request)); + + List spans = waitAndCollectSpans(2); + assertThat(spans).isNotEmpty(); + + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + // In GAX, client timeout ApiException maps to HTTP 504 + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo(504L); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isNotEmpty(); + String errorType = + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE)); + assertThat(errorType).isNotNull(); + } + } + + // F1.5: gRPC no traces emitted unless enabled. + @Test + void testTracingDisabled_grpc() throws Exception { + EchoSettings settings = createEchoSettings(false); + try (EchoClient client = EchoClient.create(settings)) { + client.echo(EchoRequest.newBuilder().setContent("test-f1-5").build()); + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).isEmpty(); + } + } + + // F1.6: gRPC T3 success case name and attributes conform to requirements. + @Test + void testT3Success_grpc() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + EchoSettings settings = createEchoSettings(false); + EchoStub stub = createStubWithServiceName(settings, tracingFactory); + + try (EchoClient client = EchoClient.create(stub)) { + client.echo(EchoRequest.newBuilder().setContent("test-f1-6").build()); + + List spans = waitAndCollectSpans(2); + assertThat(spans).isNotEmpty(); + + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + assertThat(t3Span.getKind()).isEqualTo(SpanKind.INTERNAL); + assertThat(t3Span.getName()).isEqualTo("google.showcase.v1beta1.Echo/Echo"); + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE))) + .isEqualTo("grpc"); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("OK"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.SERVER_ADDRESS_ATTRIBUTE))) + .isEqualTo(SHOWCASE_SERVER_ADDRESS); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.longKey(ObservabilityAttributes.SERVER_PORT_ATTRIBUTE))) + .isEqualTo(SHOWCASE_SERVER_PORT); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE))) + .isEqualTo(SHOWCASE_SERVICE_NAME); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNull(); + } + } + + // F1.7: gRPC T3 server failures case name and attributes conform to requirements. + @Test + void testT3ServerFailure_grpc() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + EchoSettings settings = createEchoSettings(false); + EchoStub stub = createStubWithServiceName(settings, tracingFactory); + + try (EchoClient client = EchoClient.create(stub)) { + EchoRequest request = + EchoRequest.newBuilder() + .setError( + Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT.getNumber()) + .setMessage("Server-side failure message") + .build()) + .build(); + + assertThrows(ApiException.class, () -> client.echo(request)); + + List spans = waitAndCollectSpans(2); + assertThat(spans).isNotEmpty(); + + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("INVALID_ARGUMENT"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isNotEmpty(); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } + } + + // F1.8: gRPC T3 client failures case name and attributes conform to requirements. + @Test + void testT3ClientFailure_grpc() throws Exception { + OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); + EchoSettings settings = createEchoSettings(false); + EchoStubSettings.Builder builder = + (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); + builder.setTracerFactory(tracingFactory); + builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(50L)); + EchoStub stub = new ExtendedEchoStubSettings(builder).createStub(); + + try (EchoClient client = EchoClient.create(stub)) { + BlockRequest request = + BlockRequest.newBuilder() + .setSuccess(BlockResponse.newBuilder().setContent("content").build()) + .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(3).build()) + .build(); + + assertThrows(Exception.class, () -> client.block(request)); + + List spans = waitAndCollectSpans(2); + assertThat(spans).isNotEmpty(); + + SpanData t3Span = + spans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); + + assertThat(t3Span.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + t3Span + .getAttributes() + .get( + AttributeKey.stringKey( + ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("DEADLINE_EXCEEDED"); + assertThat( + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isNotEmpty(); + String errorType = + t3Span + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE)); + assertThat(errorType).isNotNull(); + } + } + + private List waitAndCollectSpans(int minSpans) { + org.awaitility.Awaitility.await() + .atMost(java.time.Duration.ofSeconds(5)) + .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); + return spanExporter.getFinishedSpanItems(); + } + + private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { + if (isHttpJson) { + return EchoSettings.newHttpJsonBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + EchoSettings.defaultHttpJsonTransportProviderBuilder() + .setHttpTransport( + new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .build()) + .setEndpoint(SHOWCASE_HTTPJSON_ENDPOINT) + .build(); + } else { + return EchoSettings.newBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + EchoSettings.defaultGrpcTransportProviderBuilder() + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .build()) + .setEndpoint(SHOWCASE_GRPC_ENDPOINT) + .build(); + } + } + + private EchoStub createStubWithServiceName( + EchoSettings settings, OpenTelemetryTracingFactory tracingFactory) throws IOException { + EchoStubSettings.Builder builder = + (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); + builder.setTracerFactory(tracingFactory); + return new ExtendedEchoStubSettings(builder).createStub(); + } + + private static class ExtendedEchoStubSettings extends EchoStubSettings { + protected ExtendedEchoStubSettings(EchoStubSettings.Builder builder) throws IOException { + super(builder); + } + + @Override + public String getServiceName() { + return SHOWCASE_SERVICE_NAME; + } + } +} From ad7ca9c0ef39e9054f5db6148ce08b5cda1205ec Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:40:17 -0400 Subject: [PATCH 49/58] style(showcase): use simple name StatusCode in ITOtelT3Tracing --- .../google/showcase/v1beta1/it/ITOtelT3Tracing.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index 3763b24f4212..6015c2fb0016 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -51,6 +51,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -128,7 +129,7 @@ void testT3Success_httpjson() throws Exception { assertThat(t3Span.getKind()).isEqualTo(SpanKind.INTERNAL); assertThat(t3Span.getName()).isNotEmpty(); assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + .isEqualTo(StatusCode.UNSET); assertThat( t3Span .getAttributes() @@ -198,7 +199,7 @@ void testT3ServerFailure_httpjson() throws Exception { .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + .isEqualTo(StatusCode.ERROR); assertThat( t3Span .getAttributes() @@ -249,7 +250,7 @@ void testT3ClientFailure_httpjson() throws Exception { .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + .isEqualTo(StatusCode.ERROR); // In GAX, client timeout ApiException maps to HTTP 504 assertThat( t3Span @@ -303,7 +304,7 @@ void testT3Success_grpc() throws Exception { assertThat(t3Span.getKind()).isEqualTo(SpanKind.INTERNAL); assertThat(t3Span.getName()).isEqualTo("google.showcase.v1beta1.Echo/Echo"); assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + .isEqualTo(StatusCode.UNSET); assertThat( t3Span .getAttributes() @@ -369,7 +370,7 @@ void testT3ServerFailure_grpc() throws Exception { .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + .isEqualTo(StatusCode.ERROR); assertThat( t3Span .getAttributes() @@ -420,7 +421,7 @@ void testT3ClientFailure_grpc() throws Exception { .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + .isEqualTo(StatusCode.ERROR); assertThat( t3Span .getAttributes() From cc1553b286dcc49bfa450ddb4ef063233e331e2b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:05:02 -0400 Subject: [PATCH 50/58] chore(showcase): use Awaitility untilAsserted and avoid FQCN in showcase tests --- .../v1beta1/it/ITOtelT3MetricsExemplar.java | 96 ++++++++----------- .../v1beta1/it/ITOtelT3T4Hierarchy.java | 3 +- .../showcase/v1beta1/it/ITOtelT3Tracing.java | 21 ++-- 3 files changed, 52 insertions(+), 68 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index 5aa972570b22..0ea1d1924179 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -130,20 +130,27 @@ void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); - Collection metrics = waitAndCollectMetrics(); - MetricData durationMetric = - metrics.stream() - .filter(m -> m.getName().equals("gcp.client.request.duration")) - .findFirst() - .orElseThrow(() -> new AssertionError("Duration metric not found in: " + metrics)); - - HistogramPointData point = durationMetric.getHistogramData().getPoints().iterator().next(); - List exemplars = new ArrayList<>(point.getExemplars()); - assertThat(exemplars).isNotEmpty(); - - ExemplarData exemplar = exemplars.get(0); - assertThat(exemplar.getSpanContext().getTraceId()).isEqualTo(t3Span.getTraceId()); - assertThat(exemplar.getSpanContext().getSpanId()).isEqualTo(t3Span.getSpanId()); + Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .untilAsserted( + () -> { + Collection metrics = metricReader.collectAllMetrics(); + MetricData durationMetric = + metrics.stream() + .filter(m -> m.getName().equals("gcp.client.request.duration")) + .findFirst() + .orElseThrow( + () -> new AssertionError("Duration metric not found in: " + metrics)); + + HistogramPointData point = + durationMetric.getHistogramData().getPoints().iterator().next(); + List exemplars = new ArrayList<>(point.getExemplars()); + assertThat(exemplars).isNotEmpty(); + + ExemplarData exemplar = exemplars.get(0); + assertThat(exemplar.getSpanContext().getTraceId()).isEqualTo(t3Span.getTraceId()); + assertThat(exemplar.getSpanContext().getSpanId()).isEqualTo(t3Span.getSpanId()); + }); } } @@ -164,20 +171,27 @@ void testGrpc_m3ExemplarMatchesT3Span() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); - Collection metrics = waitAndCollectMetrics(); - MetricData durationMetric = - metrics.stream() - .filter(m -> m.getName().equals("gcp.client.request.duration")) - .findFirst() - .orElseThrow(() -> new AssertionError("Duration metric not found in: " + metrics)); - - HistogramPointData point = durationMetric.getHistogramData().getPoints().iterator().next(); - List exemplars = new ArrayList<>(point.getExemplars()); - assertThat(exemplars).isNotEmpty(); - - ExemplarData exemplar = exemplars.get(0); - assertThat(exemplar.getSpanContext().getTraceId()).isEqualTo(t3Span.getTraceId()); - assertThat(exemplar.getSpanContext().getSpanId()).isEqualTo(t3Span.getSpanId()); + Awaitility.await() + .atMost(Duration.ofSeconds(5)) + .untilAsserted( + () -> { + Collection metrics = metricReader.collectAllMetrics(); + MetricData durationMetric = + metrics.stream() + .filter(m -> m.getName().equals("gcp.client.request.duration")) + .findFirst() + .orElseThrow( + () -> new AssertionError("Duration metric not found in: " + metrics)); + + HistogramPointData point = + durationMetric.getHistogramData().getPoints().iterator().next(); + List exemplars = new ArrayList<>(point.getExemplars()); + assertThat(exemplars).isNotEmpty(); + + ExemplarData exemplar = exemplars.get(0); + assertThat(exemplar.getSpanContext().getTraceId()).isEqualTo(t3Span.getTraceId()); + assertThat(exemplar.getSpanContext().getSpanId()).isEqualTo(t3Span.getSpanId()); + }); } } @@ -194,32 +208,6 @@ private List waitAndCollectSpans(int minSpans) { return spanExporter.getFinishedSpanItems(); } - private Collection waitAndCollectMetrics() { - java.util.concurrent.atomic.AtomicReference> holder = - new java.util.concurrent.atomic.AtomicReference<>(); - Awaitility.await() - .atMost(Duration.ofSeconds(5)) - .until( - () -> { - Collection metrics = metricReader.collectAllMetrics(); - if (metrics.isEmpty()) { - return false; - } - for (MetricData md : metrics) { - if (md.getName().equals("gcp.client.request.duration")) { - for (HistogramPointData p : md.getHistogramData().getPoints()) { - if (!p.getExemplars().isEmpty()) { - holder.set(metrics); - return true; - } - } - } - } - return false; - }); - return holder.get(); - } - private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { if (isHttpJson) { return EchoSettings.newHttpJsonBuilder() diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java index 23714a67aef0..8c1b011324a3 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java @@ -65,6 +65,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -485,7 +486,7 @@ void testGrpc_retriesExhausted() throws Exception { } private List waitAndCollectSpans(int minSpans) { - org.awaitility.Awaitility.await() + Awaitility.await() .atMost(Duration.ofSeconds(5)) .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); return spanExporter.getFinishedSpanItems(); diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index 6015c2fb0016..1e24b7b00088 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -59,6 +59,7 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.io.IOException; import java.util.List; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -128,8 +129,7 @@ void testT3Success_httpjson() throws Exception { assertThat(t3Span.getKind()).isEqualTo(SpanKind.INTERNAL); assertThat(t3Span.getName()).isNotEmpty(); - assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(StatusCode.UNSET); + assertThat(t3Span.getStatus().getStatusCode()).isEqualTo(StatusCode.UNSET); assertThat( t3Span .getAttributes() @@ -198,8 +198,7 @@ void testT3ServerFailure_httpjson() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); - assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(StatusCode.ERROR); + assertThat(t3Span.getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); assertThat( t3Span .getAttributes() @@ -249,8 +248,7 @@ void testT3ClientFailure_httpjson() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); - assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(StatusCode.ERROR); + assertThat(t3Span.getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); // In GAX, client timeout ApiException maps to HTTP 504 assertThat( t3Span @@ -303,8 +301,7 @@ void testT3Success_grpc() throws Exception { assertThat(t3Span.getKind()).isEqualTo(SpanKind.INTERNAL); assertThat(t3Span.getName()).isEqualTo("google.showcase.v1beta1.Echo/Echo"); - assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(StatusCode.UNSET); + assertThat(t3Span.getStatus().getStatusCode()).isEqualTo(StatusCode.UNSET); assertThat( t3Span .getAttributes() @@ -369,8 +366,7 @@ void testT3ServerFailure_grpc() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); - assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(StatusCode.ERROR); + assertThat(t3Span.getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); assertThat( t3Span .getAttributes() @@ -420,8 +416,7 @@ void testT3ClientFailure_grpc() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("T3 INTERNAL span not found in: " + spans)); - assertThat(t3Span.getStatus().getStatusCode()) - .isEqualTo(StatusCode.ERROR); + assertThat(t3Span.getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); assertThat( t3Span .getAttributes() @@ -443,7 +438,7 @@ void testT3ClientFailure_grpc() throws Exception { } private List waitAndCollectSpans(int minSpans) { - org.awaitility.Awaitility.await() + Awaitility.await() .atMost(java.time.Duration.ofSeconds(5)) .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); return spanExporter.getFinishedSpanItems(); From 7d9cebbd2878895ad6a3f09c40a9358b460bcb3d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:14:04 -0400 Subject: [PATCH 51/58] chore(showcase): address review comments on try-with-resources, duration units, and non-empty points --- .../v1beta1/it/ITOtelT3MetricsExemplar.java | 12 ++++--- .../v1beta1/it/ITOtelT3T4Hierarchy.java | 32 +++++-------------- .../showcase/v1beta1/it/ITOtelT3Tracing.java | 3 +- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index 0ea1d1924179..445da61fdda6 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -142,8 +142,10 @@ void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { .orElseThrow( () -> new AssertionError("Duration metric not found in: " + metrics)); - HistogramPointData point = - durationMetric.getHistogramData().getPoints().iterator().next(); + Collection points = + durationMetric.getHistogramData().getPoints(); + assertThat(points).isNotEmpty(); + HistogramPointData point = points.iterator().next(); List exemplars = new ArrayList<>(point.getExemplars()); assertThat(exemplars).isNotEmpty(); @@ -183,8 +185,10 @@ void testGrpc_m3ExemplarMatchesT3Span() throws Exception { .orElseThrow( () -> new AssertionError("Duration metric not found in: " + metrics)); - HistogramPointData point = - durationMetric.getHistogramData().getPoints().iterator().next(); + Collection points = + durationMetric.getHistogramData().getPoints(); + assertThat(points).isNotEmpty(); + HistogramPointData point = points.iterator().next(); List exemplars = new ArrayList<>(point.getExemplars()); assertThat(exemplars).isNotEmpty(); diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java index 8c1b011324a3..70eb11599aaa 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java @@ -148,11 +148,9 @@ void testHttpJson_retrySucceeds() throws Exception { .setMaxAttempts(3) .build(); - SequenceServiceClient client = + try (SequenceServiceClient client = createSequenceClient( - true, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); - - try { + true, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE))) { client.attemptSequence( AttemptSequenceRequest.newBuilder().setName(createdSequence.getName()).build()); @@ -202,8 +200,6 @@ void testHttpJson_retrySucceeds() throws Exception { .getAttributes() .get(AttributeKey.stringKey(ObservabilityAttributes.HTTP_URL_TEMPLATE_ATTRIBUTE))) .isNotEmpty(); - } finally { - client.close(); } } @@ -246,11 +242,9 @@ void testHttpJson_retriesExhausted() throws Exception { .setMaxAttempts(2) .build(); - SequenceServiceClient client = + try (SequenceServiceClient client = createSequenceClient( - true, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); - - try { + true, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE))) { assertThrows( ApiException.class, () -> @@ -303,8 +297,6 @@ void testHttpJson_retriesExhausted() throws Exception { .getAttributes() .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) .isNotNull(); - } finally { - client.close(); } } @@ -342,11 +334,9 @@ void testGrpc_retrySucceeds() throws Exception { .setMaxAttempts(3) .build(); - SequenceServiceClient client = + try (SequenceServiceClient client = createSequenceClient( - false, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); - - try { + false, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE))) { client.attemptSequence( AttemptSequenceRequest.newBuilder().setName(createdSequence.getName()).build()); @@ -385,8 +375,6 @@ void testGrpc_retrySucceeds() throws Exception { AttributeKey.stringKey( ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) .isEqualTo("OK"); - } finally { - client.close(); } } @@ -429,11 +417,9 @@ void testGrpc_retriesExhausted() throws Exception { .setMaxAttempts(2) .build(); - SequenceServiceClient client = + try (SequenceServiceClient client = createSequenceClient( - false, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE)); - - try { + false, tracingFactory, retrySettings, ImmutableSet.of(StatusCode.Code.UNAVAILABLE))) { assertThrows( ApiException.class, () -> @@ -480,8 +466,6 @@ void testGrpc_retriesExhausted() throws Exception { .getAttributes() .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) .isNotNull(); - } finally { - client.close(); } } diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index 1e24b7b00088..4cc8e93e7327 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -59,6 +59,7 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.io.IOException; import java.util.List; +import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -439,7 +440,7 @@ void testT3ClientFailure_grpc() throws Exception { private List waitAndCollectSpans(int minSpans) { Awaitility.await() - .atMost(java.time.Duration.ofSeconds(5)) + .atMost(5, TimeUnit.SECONDS) .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); return spanExporter.getFinishedSpanItems(); } From 2fb31124af3c44ff76ed2c10603cd33bfa741bdd Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:19:26 -0400 Subject: [PATCH 52/58] chore(showcase): avoid global OpenTelemetrySdk registration and increase timeout in deadline test --- .../v1beta1/it/ITOtelT3MetricsExemplar.java | 4 +--- .../showcase/v1beta1/it/ITOtelT3T4Hierarchy.java | 5 +---- .../google/showcase/v1beta1/it/ITOtelT3Tracing.java | 13 +++++-------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index 445da61fdda6..32fb9ef367b8 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -44,7 +44,6 @@ import com.google.showcase.v1beta1.stub.EchoStub; import com.google.showcase.v1beta1.stub.EchoStubSettings; import io.grpc.ManagedChannelBuilder; -import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.metrics.ExemplarFilter; @@ -102,7 +101,7 @@ void setUp() { OpenTelemetrySdk.builder() .setTracerProvider(tracerProvider) .setMeterProvider(meterProvider) - .buildAndRegisterGlobal(); + .build(); } @AfterEach @@ -110,7 +109,6 @@ void tearDown() { if (openTelemetrySdk != null) { openTelemetrySdk.close(); } - GlobalOpenTelemetry.resetForTest(); } // F3.1: HTTP M3 metric records T3 span as exemplar diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java index 70eb11599aaa..db32cc23cb6d 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java @@ -51,7 +51,6 @@ import com.google.showcase.v1beta1.it.util.TestClientInitializer; import com.google.showcase.v1beta1.stub.SequenceServiceStubSettings; import io.grpc.ManagedChannelBuilder; -import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.sdk.OpenTelemetrySdk; @@ -92,8 +91,7 @@ void setUp() throws Exception { SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) .build(); - openTelemetrySdk = - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + openTelemetrySdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); setupGrpcClient = TestClientInitializer.createGrpcSequenceClient(); setupHttpJsonClient = TestClientInitializer.createHttpJsonSequenceClient(); @@ -110,7 +108,6 @@ void tearDown() { if (openTelemetrySdk != null) { openTelemetrySdk.close(); } - GlobalOpenTelemetry.resetForTest(); } // F2.1: HTTP T3/T4 retry succeeds (1 T3 span, 2 T4 child spans) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index 4cc8e93e7327..8ca37d72f5cb 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -48,7 +48,6 @@ import com.google.showcase.v1beta1.stub.EchoStub; import com.google.showcase.v1beta1.stub.EchoStubSettings; import io.grpc.ManagedChannelBuilder; -import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; @@ -86,8 +85,7 @@ void setUp() { SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) .build(); - openTelemetrySdk = - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + openTelemetrySdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); } @AfterEach @@ -95,7 +93,6 @@ void tearDown() { if (openTelemetrySdk != null) { openTelemetrySdk.close(); } - GlobalOpenTelemetry.resetForTest(); } // F1.1: HTTP no traces emitted unless enabled. @@ -228,14 +225,14 @@ void testT3ClientFailure_httpjson() throws Exception { EchoStubSettings.Builder builder = (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); builder.setTracerFactory(tracingFactory); - builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(50L)); + builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(1000L)); EchoStub stub = new ExtendedEchoStubSettings(builder).createStub(); try (EchoClient client = EchoClient.create(stub)) { BlockRequest request = BlockRequest.newBuilder() .setSuccess(BlockResponse.newBuilder().setContent("content").build()) - .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(3).build()) + .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(5).build()) .build(); assertThrows(Exception.class, () -> client.block(request)); @@ -396,14 +393,14 @@ void testT3ClientFailure_grpc() throws Exception { EchoStubSettings.Builder builder = (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); builder.setTracerFactory(tracingFactory); - builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(50L)); + builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(1000L)); EchoStub stub = new ExtendedEchoStubSettings(builder).createStub(); try (EchoClient client = EchoClient.create(stub)) { BlockRequest request = BlockRequest.newBuilder() .setSuccess(BlockResponse.newBuilder().setContent("content").build()) - .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(3).build()) + .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(5).build()) .build(); assertThrows(Exception.class, () -> client.block(request)); From ca95b10a131ff2f419aabccc5a4597d4de1e5638 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:28:12 -0400 Subject: [PATCH 53/58] test(showcase): use java.time.Duration in ITOtelT3Tracing --- .../com/google/showcase/v1beta1/it/ITOtelT3Tracing.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index 8ca37d72f5cb..b933bb51a8bc 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -57,13 +57,13 @@ import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.io.IOException; +import java.time.Duration; import java.util.List; import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.threeten.bp.Duration; /** Integration tests for Feature 1 (F1.1–F1.8): Client-level T3 tracing across HTTP and gRPC. */ class ITOtelT3Tracing { @@ -225,7 +225,7 @@ void testT3ClientFailure_httpjson() throws Exception { EchoStubSettings.Builder builder = (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); builder.setTracerFactory(tracingFactory); - builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(1000L)); + builder.blockSettings().setSimpleTimeoutNoRetriesDuration(Duration.ofMillis(1000L)); EchoStub stub = new ExtendedEchoStubSettings(builder).createStub(); try (EchoClient client = EchoClient.create(stub)) { @@ -393,7 +393,7 @@ void testT3ClientFailure_grpc() throws Exception { EchoStubSettings.Builder builder = (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); builder.setTracerFactory(tracingFactory); - builder.blockSettings().setSimpleTimeoutNoRetries(Duration.ofMillis(1000L)); + builder.blockSettings().setSimpleTimeoutNoRetriesDuration(Duration.ofMillis(1000L)); EchoStub stub = new ExtendedEchoStubSettings(builder).createStub(); try (EchoClient client = EchoClient.create(stub)) { From 96dc083dc8689f63affaaaa98fe5aae70811be4b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:34:47 -0400 Subject: [PATCH 54/58] test(showcase): update timeout comment in ITOtelT3Tracing --- .../java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index b933bb51a8bc..0217f897f8b2 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -221,7 +221,7 @@ void testT3ServerFailure_httpjson() throws Exception { void testT3ClientFailure_httpjson() throws Exception { OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(true); - // Configure 50ms timeout for blockCallable + // Configure 1000ms timeout for blockCallable EchoStubSettings.Builder builder = (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); builder.setTracerFactory(tracingFactory); @@ -390,6 +390,7 @@ void testT3ServerFailure_grpc() throws Exception { void testT3ClientFailure_grpc() throws Exception { OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(false); + // Configure 1000ms timeout for blockCallable EchoStubSettings.Builder builder = (EchoStubSettings.Builder) settings.getStubSettings().toBuilder(); builder.setTracerFactory(tracingFactory); From 8d4643ea21ebdae1a0a71dcd8b6d0b927fb25d7b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:42:31 -0400 Subject: [PATCH 55/58] test(showcase): address review comments on exemplar types, timeout assertions, and durations --- .../showcase/v1beta1/it/ITOtelT3MetricsExemplar.java | 5 ++--- .../com/google/showcase/v1beta1/it/ITOtelT3Tracing.java | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index 32fb9ef367b8..8e7108b77d6a 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -58,7 +58,6 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.io.IOException; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -144,7 +143,7 @@ void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { durationMetric.getHistogramData().getPoints(); assertThat(points).isNotEmpty(); HistogramPointData point = points.iterator().next(); - List exemplars = new ArrayList<>(point.getExemplars()); + List exemplars = point.getExemplars(); assertThat(exemplars).isNotEmpty(); ExemplarData exemplar = exemplars.get(0); @@ -187,7 +186,7 @@ void testGrpc_m3ExemplarMatchesT3Span() throws Exception { durationMetric.getHistogramData().getPoints(); assertThat(points).isNotEmpty(); HistogramPointData point = points.iterator().next(); - List exemplars = new ArrayList<>(point.getExemplars()); + List exemplars = point.getExemplars(); assertThat(exemplars).isNotEmpty(); ExemplarData exemplar = exemplars.get(0); diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index 0217f897f8b2..d9ecbdcf91c9 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -59,7 +59,6 @@ import java.io.IOException; import java.time.Duration; import java.util.List; -import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -235,7 +234,7 @@ void testT3ClientFailure_httpjson() throws Exception { .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(5).build()) .build(); - assertThrows(Exception.class, () -> client.block(request)); + assertThrows(ApiException.class, () -> client.block(request)); List spans = waitAndCollectSpans(2); assertThat(spans).isNotEmpty(); @@ -404,7 +403,7 @@ void testT3ClientFailure_grpc() throws Exception { .setResponseDelay(com.google.protobuf.Duration.newBuilder().setSeconds(5).build()) .build(); - assertThrows(Exception.class, () -> client.block(request)); + assertThrows(ApiException.class, () -> client.block(request)); List spans = waitAndCollectSpans(2); assertThat(spans).isNotEmpty(); @@ -438,7 +437,7 @@ void testT3ClientFailure_grpc() throws Exception { private List waitAndCollectSpans(int minSpans) { Awaitility.await() - .atMost(5, TimeUnit.SECONDS) + .atMost(Duration.ofSeconds(5)) .until(() -> spanExporter.getFinishedSpanItems().size() >= minSpans); return spanExporter.getFinishedSpanItems(); } From cd8642ac16e0978555613f1f738a361dab257c61 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:47:06 -0400 Subject: [PATCH 56/58] test(showcase): assert exact exemplar points size in ITOtelT3MetricsExemplar --- .../google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index 8e7108b77d6a..b56f245b96e7 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -141,7 +141,7 @@ void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { Collection points = durationMetric.getHistogramData().getPoints(); - assertThat(points).isNotEmpty(); + assertThat(points).hasSize(1); HistogramPointData point = points.iterator().next(); List exemplars = point.getExemplars(); assertThat(exemplars).isNotEmpty(); @@ -184,7 +184,7 @@ void testGrpc_m3ExemplarMatchesT3Span() throws Exception { Collection points = durationMetric.getHistogramData().getPoints(); - assertThat(points).isNotEmpty(); + assertThat(points).hasSize(1); HistogramPointData point = points.iterator().next(); List exemplars = point.getExemplars(); assertThat(exemplars).isNotEmpty(); From 37288a863b1946c94877f13240789ab82cd6908f Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:51:39 -0400 Subject: [PATCH 57/58] test(showcase): remove redundant certificate validation and use exact status assertions --- .../google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java | 3 +-- .../com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java | 3 +-- .../java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index b56f245b96e7..d47dc58f5ab3 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -215,8 +215,7 @@ private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { .setCredentialsProvider(NoCredentialsProvider.create()) .setTransportChannelProvider( EchoSettings.defaultHttpJsonTransportProviderBuilder() - .setHttpTransport( - new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .setHttpTransport(new NetHttpTransport.Builder().build()) .build()) .setEndpoint(SHOWCASE_HTTPJSON_ENDPOINT) .build(); diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java index db32cc23cb6d..5035fd7c4eef 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java @@ -495,8 +495,7 @@ private SequenceServiceClient createSequenceClient( settingsBuilder .setTransportChannelProvider( SequenceServiceSettings.defaultHttpJsonTransportProviderBuilder() - .setHttpTransport( - new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .setHttpTransport(new NetHttpTransport.Builder().build()) .build()) .setEndpoint(SHOWCASE_HTTPJSON_ENDPOINT); } else { diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index d9ecbdcf91c9..d812be1b600f 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -201,7 +201,7 @@ void testT3ServerFailure_httpjson() throws Exception { .getAttributes() .get( AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))) - .isAtLeast(400L); + .isEqualTo(400L); assertThat( t3Span .getAttributes() @@ -448,8 +448,7 @@ private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { .setCredentialsProvider(NoCredentialsProvider.create()) .setTransportChannelProvider( EchoSettings.defaultHttpJsonTransportProviderBuilder() - .setHttpTransport( - new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .setHttpTransport(new NetHttpTransport.Builder().build()) .build()) .setEndpoint(SHOWCASE_HTTPJSON_ENDPOINT) .build(); From 1253bdf4236d4b719b993d78de932952c0cda36b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:32:00 -0400 Subject: [PATCH 58/58] docs(showcase): add Javadoc and explanatory comments to T3 integration tests --- .../v1beta1/it/ITOtelT3MetricsExemplar.java | 37 ++++++++++++++ .../v1beta1/it/ITOtelT3T4Hierarchy.java | 48 ++++++++++++++++++ .../showcase/v1beta1/it/ITOtelT3Tracing.java | 49 +++++++++++++++++++ 3 files changed, 134 insertions(+) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java index d47dc58f5ab3..0ff3763e7fcc 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3MetricsExemplar.java @@ -113,6 +113,8 @@ void tearDown() { // F3.1: HTTP M3 metric records T3 span as exemplar @Test void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { + // Verifies that for HTTP/JSON, client request duration metrics attach exemplars + // pointing directly to the overall T3 operation span (traceId and spanId match). ApiTracerFactory compositeTracerFactory = createCompositeTracerFactory(); EchoSettings settings = createEchoSettings(true); EchoStub stub = createStubWithServiceName(settings, compositeTracerFactory); @@ -156,6 +158,8 @@ void testHttpJson_m3ExemplarMatchesT3Span() throws Exception { // F3.2: gRPC M3 metric records T3 span as exemplar @Test void testGrpc_m3ExemplarMatchesT3Span() throws Exception { + // Verifies that for gRPC, client request duration metrics attach exemplars + // pointing directly to the overall T3 operation span (traceId and spanId match). ApiTracerFactory compositeTracerFactory = createCompositeTracerFactory(); EchoSettings settings = createEchoSettings(false); EchoStub stub = createStubWithServiceName(settings, compositeTracerFactory); @@ -196,12 +200,23 @@ void testGrpc_m3ExemplarMatchesT3Span() throws Exception { } } + /** + * Creates a composite tracer factory combining both OpenTelemetry tracing and metrics factories. + * + * @return the configured {@link CompositeTracerFactory} + */ private CompositeTracerFactory createCompositeTracerFactory() { OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); OpenTelemetryMetricsFactory metricsFactory = new OpenTelemetryMetricsFactory(openTelemetrySdk); return new CompositeTracerFactory(Arrays.asList(tracingFactory, metricsFactory)); } + /** + * Waits until the in-memory span exporter records at least {@code minSpans} completed spans. + * + * @param minSpans the minimum number of spans expected + * @return the list of completed {@link SpanData} items + */ private List waitAndCollectSpans(int minSpans) { Awaitility.await() .atMost(Duration.ofSeconds(5)) @@ -209,6 +224,13 @@ private List waitAndCollectSpans(int minSpans) { return spanExporter.getFinishedSpanItems(); } + /** + * Constructs {@link EchoSettings} configured for the local Showcase test server. + * + * @param isHttpJson {@code true} for HTTP/JSON transport; {@code false} for gRPC transport + * @return the configured {@link EchoSettings} + * @throws Exception if transport provider initialization fails + */ private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { if (isHttpJson) { return EchoSettings.newHttpJsonBuilder() @@ -231,6 +253,14 @@ private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { } } + /** + * Instantiates an {@link EchoStub} with custom service name and tracer factory. + * + * @param settings the client settings to base the stub on + * @param tracerFactory the tracer factory to register with the stub + * @return the initialized {@link EchoStub} + * @throws IOException if stub creation fails + */ private EchoStub createStubWithServiceName(EchoSettings settings, ApiTracerFactory tracerFactory) throws IOException { EchoStubSettings.Builder builder = @@ -239,7 +269,14 @@ private EchoStub createStubWithServiceName(EchoSettings settings, ApiTracerFacto return new ExtendedEchoStubSettings(builder).createStub(); } + /** Extended {@link EchoStubSettings} that overrides {@link #getServiceName()} for testing. */ private static class ExtendedEchoStubSettings extends EchoStubSettings { + /** + * Constructs settings wrapping the specified builder. + * + * @param builder the settings builder + * @throws IOException if base settings construction fails + */ protected ExtendedEchoStubSettings(EchoStubSettings.Builder builder) throws IOException { super(builder); } diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java index 5035fd7c4eef..6d2773eb308f 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3T4Hierarchy.java @@ -113,6 +113,12 @@ void tearDown() { // F2.1: HTTP T3/T4 retry succeeds (1 T3 span, 2 T4 child spans) @Test void testHttpJson_retrySucceeds() throws Exception { + // Verifies HTTP/JSON retry behavior: transient failure on attempt 0 is retried and succeeds on + // attempt 1. + // Asserts: + // 1. Exactly one overall INTERNAL operation span (T3). + // 2. Both attempt spans (T4) have parent_span_id == T3.span_id. + // 3. T3 operation span aggregates the successful status (OK) and 200 HTTP code. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); // Sequence: attempt 1 -> UNAVAILABLE, attempt 2 -> OK @@ -203,6 +209,11 @@ void testHttpJson_retrySucceeds() throws Exception { // F2.2: HTTP T3/T4 retries exhausted @Test void testHttpJson_retriesExhausted() throws Exception { + // Verifies HTTP/JSON behavior when retries are exhausted after repeated failures. + // Asserts: + // 1. Exactly one overall INTERNAL operation span (T3). + // 2. All attempt spans (T4) are linked to the T3 span as parent. + // 3. T3 operation span aggregates ERROR status, 503 HTTP status, and UNAVAILABLE RPC status. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); // Sequence: 3 UNAVAILABLE responses @@ -300,6 +311,12 @@ void testHttpJson_retriesExhausted() throws Exception { // F2.3: gRPC T3/T4 retry succeeds @Test void testGrpc_retrySucceeds() throws Exception { + // Verifies gRPC retry behavior: transient failure on attempt 0 is retried and succeeds on + // attempt 1. + // Asserts: + // 1. Exactly one overall INTERNAL operation span (T3). + // 2. Both attempt spans (T4) have parent_span_id == T3.span_id. + // 3. T3 operation span aggregates the successful status (OK). OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); // Sequence: attempt 1 -> UNAVAILABLE, attempt 2 -> OK @@ -378,6 +395,11 @@ void testGrpc_retrySucceeds() throws Exception { // F2.4: gRPC T3/T4 retries exhausted @Test void testGrpc_retriesExhausted() throws Exception { + // Verifies gRPC behavior when retries are exhausted after repeated failures. + // Asserts: + // 1. Exactly one overall INTERNAL operation span (T3). + // 2. All attempt spans (T4) are linked to the T3 span as parent. + // 3. T3 operation span aggregates ERROR status and UNAVAILABLE RPC status. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); // Sequence: 3 UNAVAILABLE responses @@ -466,6 +488,12 @@ void testGrpc_retriesExhausted() throws Exception { } } + /** + * Waits until the in-memory span exporter records at least {@code minSpans} completed spans. + * + * @param minSpans the minimum number of spans expected + * @return the list of completed {@link SpanData} items + */ private List waitAndCollectSpans(int minSpans) { Awaitility.await() .atMost(Duration.ofSeconds(5)) @@ -473,6 +501,16 @@ private List waitAndCollectSpans(int minSpans) { return spanExporter.getFinishedSpanItems(); } + /** + * Constructs a {@link SequenceServiceClient} configured with custom retry settings and tracing. + * + * @param isHttpJson {@code true} for HTTP/JSON transport; {@code false} for gRPC transport + * @param tracingFactory the tracer factory to register with the client + * @param retrySettings the custom retry settings to apply to attemptSequence + * @param retryableCodes the set of status codes considered retryable + * @return the configured {@link SequenceServiceClient} + * @throws Exception if client initialization fails + */ private SequenceServiceClient createSequenceClient( boolean isHttpJson, OpenTelemetryTracingFactory tracingFactory, @@ -514,7 +552,17 @@ private SequenceServiceClient createSequenceClient( new ExtendedSequenceServiceStubSettings(stubSettingsBuilder).createStub()); } + /** + * Extended {@link SequenceServiceStubSettings} that overrides {@link #getServiceName()} for + * testing. + */ private static class ExtendedSequenceServiceStubSettings extends SequenceServiceStubSettings { + /** + * Constructs settings wrapping the specified builder. + * + * @param builder the settings builder + * @throws IOException if base settings construction fails + */ protected ExtendedSequenceServiceStubSettings(SequenceServiceStubSettings.Builder builder) throws IOException { super(builder); diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java index d812be1b600f..d751a6f26720 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelT3Tracing.java @@ -97,6 +97,8 @@ void tearDown() { // F1.1: HTTP no traces emitted unless enabled. @Test void testTracingDisabled_httpjson() throws Exception { + // Verifies that when OpenTelemetry tracing is not configured on the client settings, + // no spans are recorded for HTTP/JSON calls. EchoSettings settings = createEchoSettings(true); try (EchoClient client = EchoClient.create(settings)) { client.echo(EchoRequest.newBuilder().setContent("test-f1-1").build()); @@ -108,6 +110,10 @@ void testTracingDisabled_httpjson() throws Exception { // F1.2: HTTP T3 success case name and attributes conform to requirements. @Test void testT3Success_httpjson() throws Exception { + // Verifies that a successful HTTP/JSON call produces a T3 INTERNAL span with proper + // semantic convention attributes (http rpc.system, server.address, server.port, 200 + // http.response.status_code, + // url.template) and UNSET status. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(true); EchoStub stub = createStubWithServiceName(settings, tracingFactory); @@ -170,6 +176,8 @@ void testT3Success_httpjson() throws Exception { // F1.3: HTTP T3 server failures case name and attributes conform to requirements. @Test void testT3ServerFailure_httpjson() throws Exception { + // Verifies that a server-side error on HTTP/JSON produces a T3 INTERNAL span with ERROR status, + // 400 http.response.status_code, and error.type. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(true); EchoStub stub = createStubWithServiceName(settings, tracingFactory); @@ -218,6 +226,9 @@ void testT3ServerFailure_httpjson() throws Exception { // F1.4: HTTP T3 client failures case name and attributes conform to requirements. @Test void testT3ClientFailure_httpjson() throws Exception { + // Verifies that a client-side timeout on HTTP/JSON produces a T3 INTERNAL span with ERROR + // status, + // 504 http.response.status_code, and error.type. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(true); // Configure 1000ms timeout for blockCallable @@ -269,6 +280,8 @@ void testT3ClientFailure_httpjson() throws Exception { // F1.5: gRPC no traces emitted unless enabled. @Test void testTracingDisabled_grpc() throws Exception { + // Verifies that when OpenTelemetry tracing is not configured on the client settings, + // no spans are recorded for gRPC calls. EchoSettings settings = createEchoSettings(false); try (EchoClient client = EchoClient.create(settings)) { client.echo(EchoRequest.newBuilder().setContent("test-f1-5").build()); @@ -280,6 +293,10 @@ void testTracingDisabled_grpc() throws Exception { // F1.6: gRPC T3 success case name and attributes conform to requirements. @Test void testT3Success_grpc() throws Exception { + // Verifies that a successful gRPC call produces a T3 INTERNAL span with proper + // semantic convention attributes (grpc rpc.system, server.address, server.port, OK + // rpc.response.status_code) + // and UNSET status. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(false); EchoStub stub = createStubWithServiceName(settings, tracingFactory); @@ -338,6 +355,8 @@ void testT3Success_grpc() throws Exception { // F1.7: gRPC T3 server failures case name and attributes conform to requirements. @Test void testT3ServerFailure_grpc() throws Exception { + // Verifies that a server-side error on gRPC produces a T3 INTERNAL span with ERROR status, + // INVALID_ARGUMENT rpc.response.status_code, and error.type. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(false); EchoStub stub = createStubWithServiceName(settings, tracingFactory); @@ -387,6 +406,8 @@ void testT3ServerFailure_grpc() throws Exception { // F1.8: gRPC T3 client failures case name and attributes conform to requirements. @Test void testT3ClientFailure_grpc() throws Exception { + // Verifies that a client-side timeout on gRPC produces a T3 INTERNAL span with ERROR status, + // DEADLINE_EXCEEDED rpc.response.status_code, and error.type. OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk); EchoSettings settings = createEchoSettings(false); // Configure 1000ms timeout for blockCallable @@ -435,6 +456,12 @@ void testT3ClientFailure_grpc() throws Exception { } } + /** + * Waits until the in-memory span exporter records at least {@code minSpans} completed spans. + * + * @param minSpans the minimum number of spans expected + * @return the list of completed {@link SpanData} items + */ private List waitAndCollectSpans(int minSpans) { Awaitility.await() .atMost(Duration.ofSeconds(5)) @@ -442,6 +469,13 @@ private List waitAndCollectSpans(int minSpans) { return spanExporter.getFinishedSpanItems(); } + /** + * Constructs {@link EchoSettings} configured for the local Showcase test server. + * + * @param isHttpJson {@code true} for HTTP/JSON transport; {@code false} for gRPC transport + * @return the configured {@link EchoSettings} + * @throws Exception if transport provider initialization fails + */ private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { if (isHttpJson) { return EchoSettings.newHttpJsonBuilder() @@ -464,6 +498,14 @@ private EchoSettings createEchoSettings(boolean isHttpJson) throws Exception { } } + /** + * Instantiates an {@link EchoStub} with custom service name and tracer factory. + * + * @param settings the client settings to base the stub on + * @param tracingFactory the tracer factory to register with the stub + * @return the initialized {@link EchoStub} + * @throws IOException if stub creation fails + */ private EchoStub createStubWithServiceName( EchoSettings settings, OpenTelemetryTracingFactory tracingFactory) throws IOException { EchoStubSettings.Builder builder = @@ -472,7 +514,14 @@ private EchoStub createStubWithServiceName( return new ExtendedEchoStubSettings(builder).createStub(); } + /** Extended {@link EchoStubSettings} that overrides {@link #getServiceName()} for testing. */ private static class ExtendedEchoStubSettings extends EchoStubSettings { + /** + * Constructs settings wrapping the specified builder. + * + * @param builder the settings builder + * @throws IOException if base settings construction fails + */ protected ExtendedEchoStubSettings(EchoStubSettings.Builder builder) throws IOException { super(builder); }