diff --git a/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj b/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj index 0f6dcf941..51c4dc163 100644 --- a/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj +++ b/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj @@ -4,7 +4,7 @@ 1.0.41 - net6.0 + net8.0;net10.0 enable enable diff --git a/backend/CPS.ComplexCases.API.Integration.Tests/CPS.ComplexCases.API.Integration.Tests.csproj b/backend/CPS.ComplexCases.API.Integration.Tests/CPS.ComplexCases.API.Integration.Tests.csproj index b439aa589..2a3f73590 100644 --- a/backend/CPS.ComplexCases.API.Integration.Tests/CPS.ComplexCases.API.Integration.Tests.csproj +++ b/backend/CPS.ComplexCases.API.Integration.Tests/CPS.ComplexCases.API.Integration.Tests.csproj @@ -1,7 +1,7 @@ - net8.0 + net10.0 enable enable false @@ -23,9 +23,9 @@ - - - + + + diff --git a/backend/CPS.ComplexCases.API.Tests/CPS.ComplexCases.API.Tests.csproj b/backend/CPS.ComplexCases.API.Tests/CPS.ComplexCases.API.Tests.csproj index 49aba6c76..ef6ec5ffa 100644 --- a/backend/CPS.ComplexCases.API.Tests/CPS.ComplexCases.API.Tests.csproj +++ b/backend/CPS.ComplexCases.API.Tests/CPS.ComplexCases.API.Tests.csproj @@ -1,7 +1,7 @@ - net8.0 + net10.0 enable enable diff --git a/backend/CPS.ComplexCases.API.Tests/Unit/Middleware/ExceptionHandlingMiddlewareTests.cs b/backend/CPS.ComplexCases.API.Tests/Unit/Middleware/ExceptionHandlingMiddlewareTests.cs index 1873b2c27..a9b6b296f 100644 --- a/backend/CPS.ComplexCases.API.Tests/Unit/Middleware/ExceptionHandlingMiddlewareTests.cs +++ b/backend/CPS.ComplexCases.API.Tests/Unit/Middleware/ExceptionHandlingMiddlewareTests.cs @@ -17,12 +17,10 @@ public void MapExceptionToStatusCode_BrokenCircuitException_MapsToServiceUnavail } [Fact] - public void MapExceptionToStatusCode_TypedBrokenCircuitException_MapsToServiceUnavailable() + public void MapExceptionToStatusCode_BrokenCircuitExceptionWithMessage_MapsToServiceUnavailable() { var statusCode = ExceptionHandlingMiddleware.MapExceptionToStatusCode( - new BrokenCircuitException( - "Circuit open", - new HttpResponseMessage(HttpStatusCode.InternalServerError))); + new BrokenCircuitException("Circuit open")); Assert.Equal(HttpStatusCode.ServiceUnavailable, statusCode); } diff --git a/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj b/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj index 402cbefd0..63daceccd 100644 --- a/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj +++ b/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj @@ -1,6 +1,6 @@  - net8.0 + net10.0 v4 Exe enable @@ -25,14 +25,14 @@ - + - + - - + - + + diff --git a/backend/CPS.ComplexCases.API/Extensions/IServiceCollectionExtension.cs b/backend/CPS.ComplexCases.API/Extensions/IServiceCollectionExtension.cs index 33810ce19..3ef33290c 100644 --- a/backend/CPS.ComplexCases.API/Extensions/IServiceCollectionExtension.cs +++ b/backend/CPS.ComplexCases.API/Extensions/IServiceCollectionExtension.cs @@ -1,11 +1,11 @@ -using System.Net; using CPS.ComplexCases.API.Clients.FileTransfer; using CPS.ComplexCases.API.Domain.Configuration; +using CPS.ComplexCases.Common.Extensions; +using CPS.ComplexCases.Common.Resilience; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Polly; -using Polly.Contrib.WaitAndRetry; namespace CPS.ComplexCases.API.Extensions; @@ -25,29 +25,30 @@ public static void AddFileTransferClient(this IServiceCollection services, IConf client.BaseAddress = new Uri(options.BaseUrl); client.Timeout = TimeSpan.FromSeconds(options.RequestTimeoutSeconds); }) - .AddPolicyHandler((serviceProvider, request) => + .AddResilienceHandler("file-transfer-retry", (pipeline, context) => { - var options = serviceProvider.GetRequiredService>().Value; - return CreateRetryPolicy(options); + var options = context.ServiceProvider.GetRequiredService>().Value; + var logger = context.ServiceProvider + .GetRequiredService() + .CreateLogger("CPS.ComplexCases.API.FileTransfer"); + + // Shared helper keeps POST/PUT out of status-code retries. Circuit breaker is off — + // FileTransfer only needs the common retry policy, not fail-fast shedding. + pipeline.AddStandardHttpResilience(new HttpResilienceOptions + { + ServiceName = "FileTransfer", + RetryAttempts = options.RetryAttempts > 0 ? options.RetryAttempts : 2, + FirstRetryDelay = TimeSpan.FromSeconds( + options.FirstRetryDelaySeconds > 0 ? options.FirstRetryDelaySeconds : 1), + CircuitBreakerFailureThreshold = 0.5, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(30), + CircuitBreakerMinimumThroughput = 10, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(30), + EnableCircuitBreaker = false, + ConcurrencyLimit = 0, + }, logger); }); services.AddTransient(); } - - private static IAsyncPolicy CreateRetryPolicy(FileTransferApiOptions options) - { - var retryAttempts = options.RetryAttempts > 0 ? options.RetryAttempts : 2; - var firstRetryDelaySeconds = options.FirstRetryDelaySeconds > 0 ? options.FirstRetryDelaySeconds : 1; - - return Policy - .HandleResult(ShouldRetry) - .WaitAndRetryAsync(Backoff.DecorrelatedJitterBackoffV2( - medianFirstRetryDelay: TimeSpan.FromSeconds(firstRetryDelaySeconds), - retryCount: retryAttempts)); - } - - private static bool ShouldRetry(HttpResponseMessage response) - { - return response.StatusCode >= HttpStatusCode.InternalServerError; - } -} \ No newline at end of file +} diff --git a/backend/CPS.ComplexCases.ActivityLog.Tests/CPS.ComplexCases.ActivityLog.Tests.csproj b/backend/CPS.ComplexCases.ActivityLog.Tests/CPS.ComplexCases.ActivityLog.Tests.csproj index 49520cb5d..d99bd6e9f 100644 --- a/backend/CPS.ComplexCases.ActivityLog.Tests/CPS.ComplexCases.ActivityLog.Tests.csproj +++ b/backend/CPS.ComplexCases.ActivityLog.Tests/CPS.ComplexCases.ActivityLog.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net10.0 enable enable true diff --git a/backend/CPS.ComplexCases.ActivityLog/CPS.ComplexCases.ActivityLog.csproj b/backend/CPS.ComplexCases.ActivityLog/CPS.ComplexCases.ActivityLog.csproj index 1fca1aba9..97f259d6e 100644 --- a/backend/CPS.ComplexCases.ActivityLog/CPS.ComplexCases.ActivityLog.csproj +++ b/backend/CPS.ComplexCases.ActivityLog/CPS.ComplexCases.ActivityLog.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -10,7 +10,6 @@ - diff --git a/backend/CPS.ComplexCases.Common.Tests/CPS.ComplexCases.Common.Tests.csproj b/backend/CPS.ComplexCases.Common.Tests/CPS.ComplexCases.Common.Tests.csproj index 1e5763b4b..b9d7cfe94 100644 --- a/backend/CPS.ComplexCases.Common.Tests/CPS.ComplexCases.Common.Tests.csproj +++ b/backend/CPS.ComplexCases.Common.Tests/CPS.ComplexCases.Common.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net10.0 enable enable false diff --git a/backend/CPS.ComplexCases.Common.Tests/Extensions/ResiliencePipelineExtensionsTests.cs b/backend/CPS.ComplexCases.Common.Tests/Extensions/ResiliencePipelineExtensionsTests.cs new file mode 100644 index 000000000..d70e9d080 --- /dev/null +++ b/backend/CPS.ComplexCases.Common.Tests/Extensions/ResiliencePipelineExtensionsTests.cs @@ -0,0 +1,248 @@ +using System.Net; +using CPS.ComplexCases.Common.Extensions; +using CPS.ComplexCases.Common.Resilience; +using Microsoft.Extensions.Logging.Abstractions; +using Polly; +using Polly.CircuitBreaker; + +namespace CPS.ComplexCases.Common.Tests.Extensions; + +public class ResiliencePipelineExtensionsTests +{ + private const int MinimumThroughput = 4; + + // Retry is disabled so each call maps to exactly one circuit-breaker sample, isolating breaker behaviour. + private static ResiliencePipeline BuildPipeline( + TimeSpan? breakDuration = null, + IReadOnlyCollection? additionalRetryableStatusCodes = null) => + new ResiliencePipelineBuilder() + .AddStandardHttpResilience(new HttpResilienceOptions + { + ServiceName = "Test", + RetryAttempts = 0, + FirstRetryDelay = TimeSpan.FromMilliseconds(1), + CircuitBreakerFailureThreshold = 0.5, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(10), + CircuitBreakerMinimumThroughput = MinimumThroughput, + // Polly v8 requires a minimum break duration of 500ms. + CircuitBreakerDurationOfBreak = breakDuration ?? TimeSpan.FromMilliseconds(500), + ConcurrencyLimit = 0, + AdditionalRetryableStatusCodes = additionalRetryableStatusCodes ?? [], + }, NullLogger.Instance) + .Build(); + + private static ValueTask Respond(HttpStatusCode statusCode) => + ValueTask.FromResult(new HttpResponseMessage(statusCode)); + + [Fact] + public async Task CircuitBreaker_OpensAfterRepeatedServerErrors() + { + var pipeline = BuildPipeline(); + + for (var i = 0; i < MinimumThroughput; i++) + { + await pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.InternalServerError)); + } + + await Assert.ThrowsAnyAsync(() => + pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.InternalServerError)).AsTask()); + } + + [Fact] + public async Task CircuitBreaker_WhenDisabled_DoesNotOpenAfterRepeatedServerErrors() + { + var pipeline = new ResiliencePipelineBuilder() + .AddStandardHttpResilience(new HttpResilienceOptions + { + ServiceName = "Test", + RetryAttempts = 0, + FirstRetryDelay = TimeSpan.FromMilliseconds(1), + CircuitBreakerFailureThreshold = 0.5, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(10), + CircuitBreakerMinimumThroughput = MinimumThroughput, + CircuitBreakerDurationOfBreak = TimeSpan.FromMilliseconds(500), + EnableCircuitBreaker = false, + ConcurrencyLimit = 0, + }, NullLogger.Instance) + .Build(); + + for (var i = 0; i < MinimumThroughput * 2; i++) + { + var response = await pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.InternalServerError)); + Assert.Equal(HttpStatusCode.InternalServerError, response.StatusCode); + } + } + + [Fact] + public async Task CircuitBreaker_WhenOpen_FailsFastWithoutInvokingDelegate() + { + var pipeline = BuildPipeline(); + + for (var i = 0; i < MinimumThroughput; i++) + { + await pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.InternalServerError)); + } + + var invocations = 0; + + await Assert.ThrowsAnyAsync(() => + pipeline.ExecuteAsync(_ => + { + invocations++; + return Respond(HttpStatusCode.OK); + }).AsTask()); + + Assert.Equal(0, invocations); + } + + [Fact] + public async Task CircuitBreaker_ClosesAgainAfterBreakDurationWhenServiceRecovers() + { + var pipeline = BuildPipeline(TimeSpan.FromMilliseconds(500)); + + for (var i = 0; i < MinimumThroughput; i++) + { + await pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.InternalServerError)); + } + + await Assert.ThrowsAnyAsync(() => + pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.InternalServerError)).AsTask()); + + await Task.Delay(TimeSpan.FromMilliseconds(800)); + + var recovered = await pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.OK)); + Assert.Equal(HttpStatusCode.OK, recovered.StatusCode); + + var subsequent = await pipeline.ExecuteAsync(_ => Respond(HttpStatusCode.OK)); + Assert.Equal(HttpStatusCode.OK, subsequent.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.NotFound)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task CircuitBreaker_DoesNotOpenForNonServerErrorResponses(HttpStatusCode statusCode) + { + var pipeline = BuildPipeline(); + + for (var i = 0; i < MinimumThroughput * 2; i++) + { + await pipeline.ExecuteAsync(_ => Respond(statusCode)); + } + + var response = await pipeline.ExecuteAsync(_ => Respond(statusCode)); + Assert.Equal(statusCode, response.StatusCode); + } + + [Fact] + public async Task Retry_RetriesConfiguredStatusCodesForIdempotentMethods() + { + var pipeline = new ResiliencePipelineBuilder() + .AddStandardHttpResilience(new HttpResilienceOptions + { + ServiceName = "Test", + RetryAttempts = 2, + FirstRetryDelay = TimeSpan.FromMilliseconds(1), + CircuitBreakerFailureThreshold = 0.5, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(30), + CircuitBreakerMinimumThroughput = 100, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(30), + ConcurrencyLimit = 0, + AdditionalRetryableStatusCodes = [HttpStatusCode.TooManyRequests], + }, NullLogger.Instance) + .Build(); + + var attempts = 0; + + await pipeline.ExecuteAsync(_ => + { + attempts++; + var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test"); + return ValueTask.FromResult(new HttpResponseMessage(HttpStatusCode.TooManyRequests) { RequestMessage = request }); + }); + + Assert.Equal(3, attempts); + } + + [Fact] + public async Task Retry_DoesNotRetryNonIdempotentMethods() + { + var pipeline = new ResiliencePipelineBuilder() + .AddStandardHttpResilience(new HttpResilienceOptions + { + ServiceName = "Test", + RetryAttempts = 2, + FirstRetryDelay = TimeSpan.FromMilliseconds(1), + CircuitBreakerFailureThreshold = 0.5, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(30), + CircuitBreakerMinimumThroughput = 100, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(30), + ConcurrencyLimit = 0, + AdditionalRetryableStatusCodes = [HttpStatusCode.TooManyRequests], + }, NullLogger.Instance) + .Build(); + + var attempts = 0; + + await pipeline.ExecuteAsync(_ => + { + attempts++; + var request = new HttpRequestMessage(HttpMethod.Post, "https://example.test"); + return ValueTask.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError) { RequestMessage = request }); + }); + + Assert.Equal(1, attempts); + } + + // Regression: concurrency limiter must sit inside retry so a permit is released during backoff. + // With an outermost limiter (ConcurrencyLimit = 1), the concurrent call could not finish until + // the retrying call's blocked second attempt was released — WhenAny would time out. + [Fact] + public async Task ConcurrencyLimiter_ReleasesPermitDuringRetryBackoff() + { + var pipeline = new ResiliencePipelineBuilder() + .AddStandardHttpResilience(new HttpResilienceOptions + { + ServiceName = "Test", + RetryAttempts = 1, + // Long enough that the concurrent call is asserted while still in backoff, not on attempt 2. + FirstRetryDelay = TimeSpan.FromSeconds(2), + CircuitBreakerFailureThreshold = 0.5, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(30), + CircuitBreakerMinimumThroughput = 100, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(30), + ConcurrencyLimit = 1, + }, NullLogger.Instance) + .Build(); + + var firstAttemptReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowRetryAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempts = 0; + + var retryingCall = pipeline.ExecuteAsync(async _ => + { + if (Interlocked.Increment(ref attempts) == 1) + { + firstAttemptReturned.SetResult(); + var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test"); + return new HttpResponseMessage(HttpStatusCode.InternalServerError) { RequestMessage = request }; + } + + await allowRetryAttempt.Task; + return new HttpResponseMessage(HttpStatusCode.OK); + }).AsTask(); + + await firstAttemptReturned.Task; + // Let the pipeline release the innermost permit and enter retry backoff. + await Task.Delay(50); + + var concurrentCallTask = pipeline.ExecuteAsync(_ => + ValueTask.FromResult(new HttpResponseMessage(HttpStatusCode.OK))).AsTask(); + + var completed = await Task.WhenAny(concurrentCallTask, Task.Delay(TimeSpan.FromSeconds(2))); + Assert.Same(concurrentCallTask, completed); + Assert.Equal(HttpStatusCode.OK, (await concurrentCallTask).StatusCode); + + allowRetryAttempt.SetResult(); + await retryingCall; + } +} diff --git a/backend/CPS.ComplexCases.Common.Tests/Resilience/HttpResiliencePolicyFactoryTests.cs b/backend/CPS.ComplexCases.Common.Tests/Resilience/HttpResiliencePolicyFactoryTests.cs deleted file mode 100644 index a19ac0523..000000000 --- a/backend/CPS.ComplexCases.Common.Tests/Resilience/HttpResiliencePolicyFactoryTests.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System.Net; -using CPS.ComplexCases.Common.Resilience; -using Microsoft.Extensions.Logging.Abstractions; -using Polly; -using Polly.CircuitBreaker; - -namespace CPS.ComplexCases.Common.Tests.Resilience; - -public class HttpResiliencePolicyFactoryTests -{ - private const int MinimumThroughput = 4; - - private static IAsyncPolicy CreateBreaker(TimeSpan? durationOfBreak = null) => - HttpResiliencePolicyFactory.CreateCircuitBreakerPolicy( - NullLogger.Instance, - serviceName: "Test", - failureThreshold: 0.5, - samplingDuration: TimeSpan.FromSeconds(10), - minimumThroughput: MinimumThroughput, - durationOfBreak: durationOfBreak ?? TimeSpan.FromMilliseconds(300)); - - private static Task Respond(HttpStatusCode statusCode) => - Task.FromResult(new HttpResponseMessage(statusCode)); - - [Fact] - public async Task CircuitBreaker_OpensAfterRepeatedServerErrors() - { - var policy = CreateBreaker(); - - for (var i = 0; i < MinimumThroughput; i++) - { - await policy.ExecuteAsync(() => Respond(HttpStatusCode.InternalServerError)); - } - - await Assert.ThrowsAnyAsync(() => - policy.ExecuteAsync(() => Respond(HttpStatusCode.InternalServerError))); - } - - [Fact] - public async Task CircuitBreaker_WhenOpen_FailsFastWithoutInvokingDelegate() - { - var policy = CreateBreaker(); - - for (var i = 0; i < MinimumThroughput; i++) - { - await policy.ExecuteAsync(() => Respond(HttpStatusCode.InternalServerError)); - } - - var invocations = 0; - - await Assert.ThrowsAnyAsync(() => - policy.ExecuteAsync(() => - { - invocations++; - return Respond(HttpStatusCode.OK); - })); - - Assert.Equal(0, invocations); - } - - [Fact] - public async Task CircuitBreaker_ClosesAgainAfterBreakDurationWhenServiceRecovers() - { - var policy = CreateBreaker(TimeSpan.FromMilliseconds(300)); - - for (var i = 0; i < MinimumThroughput; i++) - { - await policy.ExecuteAsync(() => Respond(HttpStatusCode.InternalServerError)); - } - - await Assert.ThrowsAnyAsync(() => - policy.ExecuteAsync(() => Respond(HttpStatusCode.InternalServerError))); - - await Task.Delay(TimeSpan.FromMilliseconds(500)); - - var recovered = await policy.ExecuteAsync(() => Respond(HttpStatusCode.OK)); - Assert.Equal(HttpStatusCode.OK, recovered.StatusCode); - - var subsequent = await policy.ExecuteAsync(() => Respond(HttpStatusCode.OK)); - Assert.Equal(HttpStatusCode.OK, subsequent.StatusCode); - } - - [Theory] - [InlineData(HttpStatusCode.NotFound)] - [InlineData(HttpStatusCode.TooManyRequests)] - public async Task CircuitBreaker_DoesNotOpenForNonServerErrorResponses(HttpStatusCode statusCode) - { - var policy = CreateBreaker(); - - for (var i = 0; i < MinimumThroughput * 2; i++) - { - await policy.ExecuteAsync(() => Respond(statusCode)); - } - - var response = await policy.ExecuteAsync(() => Respond(statusCode)); - Assert.Equal(statusCode, response.StatusCode); - } -} diff --git a/backend/CPS.ComplexCases.Common.Tests/Resilience/SharedResiliencePolicyTests.cs b/backend/CPS.ComplexCases.Common.Tests/Resilience/SharedResiliencePolicyTests.cs deleted file mode 100644 index 0f4d77591..000000000 --- a/backend/CPS.ComplexCases.Common.Tests/Resilience/SharedResiliencePolicyTests.cs +++ /dev/null @@ -1,87 +0,0 @@ -using CPS.ComplexCases.Common.Resilience; -using Microsoft.Extensions.DependencyInjection; -using Polly; - -namespace CPS.ComplexCases.Common.Tests.Resilience; - -public class SharedResiliencePolicyTests -{ - private static ServiceProvider BuildProvider() => - new ServiceCollection().AddLogging().BuildServiceProvider(); - - [Fact] - public void GetPolicy_BuildsPolicyOnceAndReusesIt() - { - var builds = 0; - var shared = new SharedResiliencePolicy(_ => - { - builds++; - return Policy.NoOpAsync(); - }); - - using var provider = BuildProvider(); - - var first = shared.GetPolicy(provider); - var second = shared.GetPolicy(provider); - - Assert.Same(first, second); - Assert.Equal(1, builds); - } - - [Fact] - public void GetPolicy_AfterFirstProviderDisposed_DoesNotThrowAndReturnsCachedPolicy() - { - var shared = new SharedResiliencePolicy(loggerFactory => - { - // Force the same resolution the real wiring relies on, so the test fails if the provider is read after disposal. - loggerFactory.CreateLogger("test"); - return Policy.NoOpAsync(); - }); - - var firstProvider = BuildProvider(); - var cached = shared.GetPolicy(firstProvider); - - firstProvider.Dispose(); - - var secondProvider = BuildProvider(); - try - { - var afterDisposal = shared.GetPolicy(secondProvider); - Assert.Same(cached, afterDisposal); - } - finally - { - secondProvider.Dispose(); - } - } - - [Fact] - public void GetPolicy_MultiClientOrdering_SecondClientResolvesAfterFirstScopeDisposed() - { - // Simulates two clients in one registration, each with its own holder, as produced by AddResiliencePolicyHandler. - var clientA = new SharedResiliencePolicy(loggerFactory => - { - loggerFactory.CreateLogger("clientA"); - return Policy.NoOpAsync(); - }); - var clientB = new SharedResiliencePolicy(loggerFactory => - { - loggerFactory.CreateLogger("clientB"); - return Policy.NoOpAsync(); - }); - - // Client A runs first and captures its (then live) provider scope. - var providerA = BuildProvider(); - var policyA = clientA.GetPolicy(providerA); - - // That handler scope rotates and is disposed before client B ever runs. - providerA.Dispose(); - - // Client B's first call must resolve against its own live provider, not a retained disposed one. - using var providerB = BuildProvider(); - var exception = Record.Exception(() => clientB.GetPolicy(providerB)); - - Assert.Null(exception); - Assert.NotNull(policyA); - } -} diff --git a/backend/CPS.ComplexCases.Common/CPS.ComplexCases.Common.csproj b/backend/CPS.ComplexCases.Common/CPS.ComplexCases.Common.csproj index 29fabb00e..68aa84824 100644 --- a/backend/CPS.ComplexCases.Common/CPS.ComplexCases.Common.csproj +++ b/backend/CPS.ComplexCases.Common/CPS.ComplexCases.Common.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -10,15 +10,13 @@ + - - - diff --git a/backend/CPS.ComplexCases.Common/Extensions/ResiliencePipelineExtensions.cs b/backend/CPS.ComplexCases.Common/Extensions/ResiliencePipelineExtensions.cs new file mode 100644 index 000000000..3aa1afbf2 --- /dev/null +++ b/backend/CPS.ComplexCases.Common/Extensions/ResiliencePipelineExtensions.cs @@ -0,0 +1,132 @@ +using System.Net; +using CPS.ComplexCases.Common.Resilience; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http.Resilience; +using Microsoft.Extensions.Logging; +using Polly; + +namespace CPS.ComplexCases.Common.Extensions; + +public static class ResiliencePipelineExtensions +{ + // Builds the standard resilience-handler configuration delegate shared by every service client. + // Each caller supplies only the logger category and its tuned options; the logger creation and + // pipeline wiring are identical, so they live here to avoid duplicating them per project. + public static Action, ResilienceHandlerContext> + ConfigureStandardResilience(string loggerCategory, HttpResilienceOptions options) => + (pipeline, context) => + { + var logger = context.ServiceProvider + .GetRequiredService() + .CreateLogger(loggerCategory); + + pipeline.AddStandardHttpResilience(options, logger); + }; + + // Configures a standard HTTP resilience pipeline built on Microsoft.Extensions.Http.Resilience + // (Polly v8). Strategies are added outer-to-inner: retry, then optional circuit breaker, then + // concurrency limiter. Retry sits outside the breaker so a retry attempt re-enters the (possibly + // open) circuit and fails fast instead of bypassing it. The concurrency limiter sits innermost so + // a permit is held only for a single attempt and released during retry backoff — matching the + // previous Polly v7 WrapAsync(retry, breaker, bulkhead) ordering. An outermost limiter with + // queueLimit: int.MaxValue would hold permits across the full retry+backoff window and stall + // throughput under a burst of transient 5xx. Set EnableCircuitBreaker = false when only the + // shared retry (and optional concurrency) semantics are required. + public static ResiliencePipelineBuilder AddStandardHttpResilience( + this ResiliencePipelineBuilder pipeline, + HttpResilienceOptions options, + ILogger logger) + { + // https://learn.microsoft.com/en-us/dotnet/core/resilience/http-resilience + if (options.RetryAttempts > 0) + { + pipeline.AddRetry(new HttpRetryStrategyOptions + { + MaxRetryAttempts = options.RetryAttempts, + Delay = options.FirstRetryDelay, + BackoffType = DelayBackoffType.Exponential, + UseJitter = true, + ShouldHandle = args => + { + // Connection failures are retried regardless of method, since there is no response to + // inspect. POST/PUT are only excluded on the status-code path below. + if (options.RetryOnConnectionFailure && args.Outcome.Exception is HttpRequestException) + { + return ValueTask.FromResult(true); + } + + if (args.Outcome.Result is null) + { + return ValueTask.FromResult(false); + } + + var response = args.Outcome.Result; + var isRetryableStatus = response.StatusCode >= HttpStatusCode.InternalServerError + || options.AdditionalRetryableStatusCodes.Contains(response.StatusCode); + + return ValueTask.FromResult(isRetryableStatus && ExcludesPostAndPut(response)); + } + }); + } + + // Only "service is down" signals (5xx and connection failures) trip the breaker. Non-health + // signals such as 404 or 429 are deliberately excluded even when retry handles them. + if (options.EnableCircuitBreaker) + { + pipeline.AddCircuitBreaker(new HttpCircuitBreakerStrategyOptions + { + FailureRatio = options.CircuitBreakerFailureThreshold, + SamplingDuration = options.CircuitBreakerSamplingDuration, + MinimumThroughput = options.CircuitBreakerMinimumThroughput, + BreakDuration = options.CircuitBreakerDurationOfBreak, + ShouldHandle = args => + { + if (args.Outcome.Exception is HttpRequestException) + { + return ValueTask.FromResult(true); + } + + if (args.Outcome.Result is null) + { + return ValueTask.FromResult(false); + } + + return ValueTask.FromResult(args.Outcome.Result.StatusCode >= HttpStatusCode.InternalServerError); + }, + OnOpened = args => + { + logger.LogError( + args.Outcome.Exception, + "{ServiceName} circuit opened for {BreakDelaySeconds}s after status {StatusCode}.", + options.ServiceName, + args.BreakDuration.TotalSeconds, + args.Outcome.Result?.StatusCode); + return default; + }, + OnClosed = _ => + { + logger.LogInformation("{ServiceName} circuit reset; calls are flowing again.", options.ServiceName); + return default; + }, + OnHalfOpened = _ => + { + logger.LogInformation("{ServiceName} circuit half-open; testing the next call.", options.ServiceName); + return default; + } + }); + } + + // Innermost: permit held only during the attempt itself, freed while waiting to retry. + if (options.ConcurrencyLimit > 0) + { + pipeline.AddConcurrencyLimiter(permitLimit: options.ConcurrencyLimit, queueLimit: int.MaxValue); + } + + return pipeline; + } + + // Retries are only safe for idempotent methods, so POST and PUT are excluded. + private static bool ExcludesPostAndPut(HttpResponseMessage response) => + response.RequestMessage?.Method != HttpMethod.Post + && response.RequestMessage?.Method != HttpMethod.Put; +} diff --git a/backend/CPS.ComplexCases.Common/Resilience/HttpResilienceOptions.cs b/backend/CPS.ComplexCases.Common/Resilience/HttpResilienceOptions.cs index 5099a38a9..e156298d5 100644 --- a/backend/CPS.ComplexCases.Common/Resilience/HttpResilienceOptions.cs +++ b/backend/CPS.ComplexCases.Common/Resilience/HttpResilienceOptions.cs @@ -1,3 +1,5 @@ +using System.Net; + namespace CPS.ComplexCases.Common.Resilience; public sealed record HttpResilienceOptions @@ -9,5 +11,22 @@ public sealed record HttpResilienceOptions public required TimeSpan CircuitBreakerSamplingDuration { get; init; } public required int CircuitBreakerMinimumThroughput { get; init; } public required TimeSpan CircuitBreakerDurationOfBreak { get; init; } - public required int BulkheadMaxParallelization { get; init; } + + // When false, the circuit breaker is omitted. Used by clients that only need the shared retry + // (and optional concurrency) semantics — e.g. FileTransfer — without fail-fast shedding. + public bool EnableCircuitBreaker { get; init; } = true; + + // Maximum number of concurrent requests allowed through to the service. Set to 0 to disable the + // concurrency limiter (e.g. for low-volume request/response services). + public int ConcurrencyLimit { get; init; } + + // Retry connection-level failures (HttpRequestException) in addition to retryable status codes. + // The rate-limited services rely on status-code retries only, whereas MDS (DDEI) also retries + // transient connection failures. + public bool RetryOnConnectionFailure { get; init; } + + // Status codes (in addition to 5xx) that should be retried, e.g. 404 for MDS or 429 for + // rate-limited services. POST/PUT are always excluded because retries are only safe for + // idempotent methods. + public IReadOnlyCollection AdditionalRetryableStatusCodes { get; init; } = []; } diff --git a/backend/CPS.ComplexCases.Common/Resilience/HttpResiliencePolicyFactory.cs b/backend/CPS.ComplexCases.Common/Resilience/HttpResiliencePolicyFactory.cs deleted file mode 100644 index 3dfc45cc7..000000000 --- a/backend/CPS.ComplexCases.Common/Resilience/HttpResiliencePolicyFactory.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Net; -using Microsoft.Extensions.Logging; -using Polly; -using Polly.Contrib.WaitAndRetry; - -namespace CPS.ComplexCases.Common.Resilience; - -public static class HttpResiliencePolicyFactory -{ - // Retry uses decorrelated jitter backoff to spread out retries and avoid synchronised retry storms. - public static IAsyncPolicy CreateRetryPolicy( - int retryAttempts, - TimeSpan firstRetryDelay, - Func shouldRetry, - bool handleHttpRequestException) - { - // https://learn.microsoft.com/en-us/dotnet/architecture/microservices/implement-resilient-applications/implement-http-call-retries-exponential-backoff-polly#add-a-jitter-strategy-to-the-retry-policy - var delay = Backoff.DecorrelatedJitterBackoffV2( - medianFirstRetryDelay: firstRetryDelay, - retryCount: retryAttempts); - - var builder = handleHttpRequestException - ? Policy.Handle().OrResult(r => shouldRetry(r)) - : Policy.HandleResult(r => shouldRetry(r)); - - return builder.WaitAndRetryAsync(delay); - } - - // Only "service is down" signals (5xx and connection failures) trip the breaker. Non-health signals - // such as 404 or 429 are deliberately excluded even when retry handles them. - public static IAsyncPolicy CreateCircuitBreakerPolicy( - ILogger logger, - string serviceName, - double failureThreshold, - TimeSpan samplingDuration, - int minimumThroughput, - TimeSpan durationOfBreak) - { - return Policy - .Handle() - .OrResult(r => r.StatusCode >= HttpStatusCode.InternalServerError) - .AdvancedCircuitBreakerAsync( - failureThreshold, - samplingDuration, - minimumThroughput, - durationOfBreak, - onBreak: (outcome, breakDelay) => logger.LogError( - outcome.Exception, - "{ServiceName} circuit opened for {BreakDelaySeconds}s after status {StatusCode}.", - serviceName, - breakDelay.TotalSeconds, - outcome.Result?.StatusCode), - onReset: () => logger.LogInformation("{ServiceName} circuit reset; calls are flowing again.", serviceName), - onHalfOpen: () => logger.LogInformation("{ServiceName} circuit half-open; testing the next call.", serviceName)); - } - - public static IAsyncPolicy CreateBulkheadPolicy( - int maxParallelization, - int maxQueuingActions = int.MaxValue) - { - return Policy.BulkheadAsync(maxParallelization, maxQueuingActions); - } - - // Standard policy for rate-limited HTTP services (e.g. Egress, NetApp) 5xx and 429 are retried - // (excluding POST/PUT), but only 5xx and connection failures trip the breaker since 429 is expected - // rate limiting rather than a service outage. - public static IAsyncPolicy CreateRateLimitedResiliencePolicy( - ILogger logger, - HttpResilienceOptions options) - { - static bool shouldRetry(HttpResponseMessage response) => - (response.StatusCode >= HttpStatusCode.InternalServerError - || response.StatusCode == HttpStatusCode.TooManyRequests) - && ExcludesPostAndPut(response); - - var retryPolicy = CreateRetryPolicy( - options.RetryAttempts, - options.FirstRetryDelay, - shouldRetry, - handleHttpRequestException: false); - - var circuitBreaker = CreateCircuitBreakerPolicy( - logger, - options.ServiceName, - options.CircuitBreakerFailureThreshold, - options.CircuitBreakerSamplingDuration, - options.CircuitBreakerMinimumThroughput, - options.CircuitBreakerDurationOfBreak); - - var bulkheadPolicy = CreateBulkheadPolicy(options.BulkheadMaxParallelization); - - return Policy.WrapAsync(retryPolicy, circuitBreaker, bulkheadPolicy); - } - - // Retries are only safe for idempotent methods, so POST and PUT are excluded. - public static bool ExcludesPostAndPut(HttpResponseMessage response) => - response.RequestMessage?.Method != HttpMethod.Post - && response.RequestMessage?.Method != HttpMethod.Put; -} diff --git a/backend/CPS.ComplexCases.Common/Resilience/ResiliencePolicyHttpClientBuilderExtensions.cs b/backend/CPS.ComplexCases.Common/Resilience/ResiliencePolicyHttpClientBuilderExtensions.cs deleted file mode 100644 index af015ede3..000000000 --- a/backend/CPS.ComplexCases.Common/Resilience/ResiliencePolicyHttpClientBuilderExtensions.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Polly; - -namespace CPS.ComplexCases.Common.Resilience; - -public static class ResiliencePolicyHttpClientBuilderExtensions -{ - // Wires a circuit-breaker/resilience policy handler that builds the policy once from the first live - // service provider and caches it per client, without ever retaining the (scoped) provider. - public static IHttpClientBuilder AddResiliencePolicyHandler( - this IHttpClientBuilder builder, - Func> policyFactory) - { - var sharedPolicy = new SharedResiliencePolicy(policyFactory); - return builder.AddPolicyHandler((sp, _) => sharedPolicy.GetPolicy(sp)); - } -} diff --git a/backend/CPS.ComplexCases.Common/Resilience/SharedResiliencePolicy.cs b/backend/CPS.ComplexCases.Common/Resilience/SharedResiliencePolicy.cs deleted file mode 100644 index 659f73c20..000000000 --- a/backend/CPS.ComplexCases.Common/Resilience/SharedResiliencePolicy.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Polly; - -namespace CPS.ComplexCases.Common.Resilience; - -// Builds a resilience policy once from the first live IServiceProvider and caches it, so circuit-breaker -// state is shared across all calls for a single client (one breaker per client). The provider is only -// used inside the call to resolve the singleton ILoggerFactory and is never retained, which avoids reading -// a disposed scope after an HttpClient handler rotation. -public sealed class SharedResiliencePolicy -{ - private readonly Func> _policyFactory; - private readonly object _gate = new(); - // volatile so the fast-path read outside the lock cannot observe a non-null reference - private volatile IAsyncPolicy? _policy; - - public SharedResiliencePolicy(Func> policyFactory) - { - _policyFactory = policyFactory ?? throw new ArgumentNullException(nameof(policyFactory)); - } - - public IAsyncPolicy GetPolicy(IServiceProvider serviceProvider) - { - if (_policy is not null) - { - return _policy; - } - - lock (_gate) - { - _policy ??= _policyFactory(serviceProvider.GetRequiredService()); - } - - return _policy; - } -} diff --git a/backend/CPS.ComplexCases.DDEI.Tests/CPS.ComplexCases.DDEI.Tests.csproj b/backend/CPS.ComplexCases.DDEI.Tests/CPS.ComplexCases.DDEI.Tests.csproj index 63f03498b..cf4bcce20 100644 --- a/backend/CPS.ComplexCases.DDEI.Tests/CPS.ComplexCases.DDEI.Tests.csproj +++ b/backend/CPS.ComplexCases.DDEI.Tests/CPS.ComplexCases.DDEI.Tests.csproj @@ -1,7 +1,7 @@ - net8.0 + net10.0 enable enable diff --git a/backend/CPS.ComplexCases.DDEI.WireMock/CPS.ComplexCases.DDEI.WireMock.csproj b/backend/CPS.ComplexCases.DDEI.WireMock/CPS.ComplexCases.DDEI.WireMock.csproj index 6f074ef42..691e27212 100644 --- a/backend/CPS.ComplexCases.DDEI.WireMock/CPS.ComplexCases.DDEI.WireMock.csproj +++ b/backend/CPS.ComplexCases.DDEI.WireMock/CPS.ComplexCases.DDEI.WireMock.csproj @@ -1,7 +1,7 @@  - net8.0 + net10.0 enable enable true @@ -10,6 +10,8 @@ + + diff --git a/backend/CPS.ComplexCases.DDEI/CPS.ComplexCases.DDEI.csproj b/backend/CPS.ComplexCases.DDEI/CPS.ComplexCases.DDEI.csproj index 24ba4806f..6b756993c 100644 --- a/backend/CPS.ComplexCases.DDEI/CPS.ComplexCases.DDEI.csproj +++ b/backend/CPS.ComplexCases.DDEI/CPS.ComplexCases.DDEI.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -11,9 +11,7 @@ - - - + diff --git a/backend/CPS.ComplexCases.DDEI/Extensions/IServiceCollectionExtension.cs b/backend/CPS.ComplexCases.DDEI/Extensions/IServiceCollectionExtension.cs index 5dd25c117..d9d991329 100644 --- a/backend/CPS.ComplexCases.DDEI/Extensions/IServiceCollectionExtension.cs +++ b/backend/CPS.ComplexCases.DDEI/Extensions/IServiceCollectionExtension.cs @@ -1,16 +1,15 @@ using System.Net; using System.Net.Http.Headers; +using CPS.ComplexCases.Common.Extensions; +using CPS.ComplexCases.Common.Resilience; using CPS.ComplexCases.DDEI.Client; using CPS.ComplexCases.DDEI.Factories; using CPS.ComplexCases.DDEI.Mappers; using CPS.ComplexCases.DDEI.Services; -using CPS.ComplexCases.Common.Resilience; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Polly; namespace CPS.ComplexCases.DDEI.Extensions; @@ -30,9 +29,28 @@ public static void AddDdeiClient(this IServiceCollection services, IConfiguratio { services.Configure(configuration.GetSection(nameof(DDEIOptions))); services.AddTransient(); + + // A 404 is retried (MDS occasionally returns one transiently) but is not a health signal, so it is + // deliberately excluded from the breaker. Connection failures are also retried for this service. + var configureResilience = ResiliencePipelineExtensions.ConfigureStandardResilience( + "CPS.ComplexCases.DDEI.CircuitBreaker", + new HttpResilienceOptions + { + ServiceName = "MDS (DDEI)", + RetryAttempts = RetryAttempts, + FirstRetryDelay = TimeSpan.FromSeconds(FirstRetryDelaySeconds), + CircuitBreakerFailureThreshold = CircuitBreakerFailureThreshold, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(CircuitBreakerSamplingDurationSeconds), + CircuitBreakerMinimumThroughput = CircuitBreakerMinimumThroughput, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(CircuitBreakerDurationOfBreakSeconds), + ConcurrencyLimit = 0, + RetryOnConnectionFailure = true, + AdditionalRetryableStatusCodes = [HttpStatusCode.NotFound], + }); + services.AddHttpClient(AddDdeiClient) .SetHandlerLifetime(TimeSpan.FromMinutes(5)) - .AddResiliencePolicyHandler(GetResiliencePolicy); + .AddResilienceHandler("ddei-resilience", configureResilience); services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -52,33 +70,4 @@ internal static void AddDdeiClient(IServiceProvider configuration, HttpClient cl client.DefaultRequestHeaders.Add(DDEIOptions.DevtunnelTokenKey, opts.DevtunnelToken); } } - - // Retry is kept outermost so a retry attempt re-enters the (possibly open) circuit and fails fast - // instead of bypassing the breaker. - internal static IAsyncPolicy GetResiliencePolicy(ILoggerFactory loggerFactory) - { - var logger = loggerFactory.CreateLogger("CPS.ComplexCases.DDEI.CircuitBreaker"); - - // A 404 is retried but is not a health signal, so it is deliberately excluded from the breaker. - static bool shouldRetry(HttpResponseMessage response) => - (response.StatusCode >= HttpStatusCode.InternalServerError - || response.StatusCode == HttpStatusCode.NotFound) - && HttpResiliencePolicyFactory.ExcludesPostAndPut(response); - - var retryPolicy = HttpResiliencePolicyFactory.CreateRetryPolicy( - RetryAttempts, - TimeSpan.FromSeconds(FirstRetryDelaySeconds), - shouldRetry, - handleHttpRequestException: true); - - var circuitBreaker = HttpResiliencePolicyFactory.CreateCircuitBreakerPolicy( - logger, - "MDS (DDEI)", - CircuitBreakerFailureThreshold, - TimeSpan.FromSeconds(CircuitBreakerSamplingDurationSeconds), - CircuitBreakerMinimumThroughput, - TimeSpan.FromSeconds(CircuitBreakerDurationOfBreakSeconds)); - - return Policy.WrapAsync(retryPolicy, circuitBreaker); - } -} \ No newline at end of file +} diff --git a/backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj b/backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj index db354c750..4a06ed09e 100644 --- a/backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj +++ b/backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj @@ -1,7 +1,7 @@  Library - net8.0 + net10.0 enable enable true @@ -12,15 +12,14 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + - - + diff --git a/backend/CPS.ComplexCases.Egress.Tests/CPS.ComplexCases.Egress.Tests.csproj b/backend/CPS.ComplexCases.Egress.Tests/CPS.ComplexCases.Egress.Tests.csproj index 3cdbf7a7c..6c2985262 100644 --- a/backend/CPS.ComplexCases.Egress.Tests/CPS.ComplexCases.Egress.Tests.csproj +++ b/backend/CPS.ComplexCases.Egress.Tests/CPS.ComplexCases.Egress.Tests.csproj @@ -1,7 +1,7 @@ - net8.0 + net10.0 enable enable diff --git a/backend/CPS.ComplexCases.Egress.Tests/Unit/ResiliencePolicyOrderingTests.cs b/backend/CPS.ComplexCases.Egress.Tests/Unit/ResiliencePolicyOrderingTests.cs deleted file mode 100644 index 5c371d9da..000000000 --- a/backend/CPS.ComplexCases.Egress.Tests/Unit/ResiliencePolicyOrderingTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -using CPS.ComplexCases.Common.Resilience; -using CPS.ComplexCases.Egress.Extensions; -using Microsoft.Extensions.DependencyInjection; - -namespace CPS.ComplexCases.Egress.Tests.Unit; - -// Reproduces the registration ordering that previously threw ObjectDisposedException: -// EgressClient and EgressStorageClient each get their own policy holder, so the second -// client to run must resolve against its own live provider, not a retained, disposed scope. -public class ResiliencePolicyOrderingTests -{ - private static ServiceProvider BuildProvider() => - new ServiceCollection().AddLogging().BuildServiceProvider(); - - [Fact] - public void SecondClientResolvesPolicy_AfterFirstClientScopeDisposed() - { - var firstClient = new SharedResiliencePolicy(IServiceCollectionExtension.GetResiliencePolicy); - var secondClient = new SharedResiliencePolicy(IServiceCollectionExtension.GetResiliencePolicy); - - var firstProvider = BuildProvider(); - var firstPolicy = firstClient.GetPolicy(firstProvider); - - // The first client's handler scope rotates and is disposed before the second client first runs. - firstProvider.Dispose(); - - using var secondProvider = BuildProvider(); - var exception = Record.Exception(() => secondClient.GetPolicy(secondProvider)); - - Assert.Null(exception); - Assert.NotNull(firstPolicy); - } -} diff --git a/backend/CPS.ComplexCases.Egress.WireMock/CPS.ComplexCases.Egress.WireMock.csproj b/backend/CPS.ComplexCases.Egress.WireMock/CPS.ComplexCases.Egress.WireMock.csproj index 1231cde0b..f6944b330 100644 --- a/backend/CPS.ComplexCases.Egress.WireMock/CPS.ComplexCases.Egress.WireMock.csproj +++ b/backend/CPS.ComplexCases.Egress.WireMock/CPS.ComplexCases.Egress.WireMock.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -10,6 +10,8 @@ + + diff --git a/backend/CPS.ComplexCases.Egress/CPS.ComplexCases.Egress.csproj b/backend/CPS.ComplexCases.Egress/CPS.ComplexCases.Egress.csproj index a6c1d841e..651415b4f 100644 --- a/backend/CPS.ComplexCases.Egress/CPS.ComplexCases.Egress.csproj +++ b/backend/CPS.ComplexCases.Egress/CPS.ComplexCases.Egress.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -11,8 +11,7 @@ - - + diff --git a/backend/CPS.ComplexCases.Egress/Extensions/IServiceCollectionExtension.cs b/backend/CPS.ComplexCases.Egress/Extensions/IServiceCollectionExtension.cs index a0e1a6e0b..72223e09a 100644 --- a/backend/CPS.ComplexCases.Egress/Extensions/IServiceCollectionExtension.cs +++ b/backend/CPS.ComplexCases.Egress/Extensions/IServiceCollectionExtension.cs @@ -1,11 +1,11 @@ +using System.Net; +using CPS.ComplexCases.Common.Extensions; +using CPS.ComplexCases.Common.Resilience; using CPS.ComplexCases.Egress.Client; using CPS.ComplexCases.Egress.Factories; using CPS.ComplexCases.Egress.Models; -using CPS.ComplexCases.Common.Resilience; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Polly; namespace CPS.ComplexCases.Egress.Extensions; @@ -13,6 +13,7 @@ public static class IServiceCollectionExtension { private const int RetryAttempts = 3; private const int FirstRetryDelaySeconds = 1; + private const int ConcurrencyLimit = 30; // Egress rate-limits us with 429s (excluded from the breaker), so a slightly longer sampling window // avoids tripping on normal throttling while still catching a genuinely failing service. @@ -26,6 +27,22 @@ public static void AddEgressClient(this IServiceCollection services, IConfigurat services.AddTransient(); services.AddTransient(); services.Configure(configuration.GetSection("EgressOptions")); + + var configureResilience = ResiliencePipelineExtensions.ConfigureStandardResilience( + "CPS.ComplexCases.Egress.CircuitBreaker", + new HttpResilienceOptions + { + ServiceName = "Egress", + RetryAttempts = RetryAttempts, + FirstRetryDelay = TimeSpan.FromSeconds(FirstRetryDelaySeconds), + CircuitBreakerFailureThreshold = CircuitBreakerFailureThreshold, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(CircuitBreakerSamplingDurationSeconds), + CircuitBreakerMinimumThroughput = CircuitBreakerMinimumThroughput, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(CircuitBreakerDurationOfBreakSeconds), + ConcurrencyLimit = ConcurrencyLimit, + AdditionalRetryableStatusCodes = [HttpStatusCode.TooManyRequests], + }); + services.AddHttpClient(client => { var egressServiceUrl = configuration["EgressOptions:Url"]; @@ -37,7 +54,7 @@ public static void AddEgressClient(this IServiceCollection services, IConfigurat client.Timeout = TimeSpan.FromSeconds(configuration.GetValue("EgressOptions:ManagementTimeoutSeconds", 100)); }) .SetHandlerLifetime(TimeSpan.FromMinutes(5)) - .AddResiliencePolicyHandler(GetResiliencePolicy); + .AddResilienceHandler("egress-resilience", configureResilience); services.AddHttpClient(client => { @@ -54,23 +71,6 @@ public static void AddEgressClient(this IServiceCollection services, IConfigurat client.Timeout = Timeout.InfiniteTimeSpan; }) .SetHandlerLifetime(TimeSpan.FromMinutes(5)) - .AddResiliencePolicyHandler(GetResiliencePolicy); - } - - internal static IAsyncPolicy GetResiliencePolicy(ILoggerFactory loggerFactory) - { - var logger = loggerFactory.CreateLogger("CPS.ComplexCases.Egress.CircuitBreaker"); - - return HttpResiliencePolicyFactory.CreateRateLimitedResiliencePolicy(logger, new HttpResilienceOptions - { - ServiceName = "Egress", - RetryAttempts = RetryAttempts, - FirstRetryDelay = TimeSpan.FromSeconds(FirstRetryDelaySeconds), - CircuitBreakerFailureThreshold = CircuitBreakerFailureThreshold, - CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(CircuitBreakerSamplingDurationSeconds), - CircuitBreakerMinimumThroughput = CircuitBreakerMinimumThroughput, - CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(CircuitBreakerDurationOfBreakSeconds), - BulkheadMaxParallelization = 30, - }); + .AddResilienceHandler("egress-storage-resilience", configureResilience); } } \ No newline at end of file diff --git a/backend/CPS.ComplexCases.FileTransfer.API.Tests/CPS.ComplexCases.FileTransfer.API.Tests.csproj b/backend/CPS.ComplexCases.FileTransfer.API.Tests/CPS.ComplexCases.FileTransfer.API.Tests.csproj index 4c0c91a41..6f0b26476 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API.Tests/CPS.ComplexCases.FileTransfer.API.Tests.csproj +++ b/backend/CPS.ComplexCases.FileTransfer.API.Tests/CPS.ComplexCases.FileTransfer.API.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net10.0 enable enable false diff --git a/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj b/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj index 5741316c7..9c1feb36b 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj +++ b/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 v4 Exe enable @@ -12,15 +12,15 @@ - + - + - - + + diff --git a/backend/CPS.ComplexCases.NetApp.Tests/CPS.ComplexCases.NetApp.Tests.csproj b/backend/CPS.ComplexCases.NetApp.Tests/CPS.ComplexCases.NetApp.Tests.csproj index 8fb1cb11f..25483817b 100644 --- a/backend/CPS.ComplexCases.NetApp.Tests/CPS.ComplexCases.NetApp.Tests.csproj +++ b/backend/CPS.ComplexCases.NetApp.Tests/CPS.ComplexCases.NetApp.Tests.csproj @@ -1,7 +1,7 @@ - net8.0 + net10.0 enable enable diff --git a/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppClientTests.cs b/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppClientTests.cs index e0e31fb42..6c670fe40 100644 --- a/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppClientTests.cs +++ b/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppClientTests.cs @@ -1,5 +1,6 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; +using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Text; using Microsoft.Extensions.Logging; @@ -159,7 +160,9 @@ public NetAppClientTests() _netAppS3HttpArgFactory = new NetAppS3HttpArgFactory(); - var testCert = new X509Certificate2([]); + using var rsa = RSA.Create(2048); + var certReq = new CertificateRequest("CN=TestCA", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var testCert = certReq.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddYears(1)); var testCertCollection = new X509Certificate2Collection { testCert }; _mockNetAppCertFactory diff --git a/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppStorageClientTests.cs b/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppStorageClientTests.cs index 5df86d106..6e87514b2 100644 --- a/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppStorageClientTests.cs +++ b/backend/CPS.ComplexCases.NetApp.Tests/Integration/NetAppStorageClientTests.cs @@ -40,7 +40,6 @@ public class NetAppStorageClientTests : IDisposable public NetAppStorageClientTests() { - System.Net.ServicePointManager.ServerCertificateValidationCallback = (sender, cert, chain, sslPolicyErrors) => true; _server = WireMockServer.Start(new WireMockServerSettings { UseSSL = true diff --git a/backend/CPS.ComplexCases.NetApp.Tests/Unit/Extensions/ResiliencePolicyOrderingTests.cs b/backend/CPS.ComplexCases.NetApp.Tests/Unit/Extensions/ResiliencePolicyOrderingTests.cs deleted file mode 100644 index 0601d97d4..000000000 --- a/backend/CPS.ComplexCases.NetApp.Tests/Unit/Extensions/ResiliencePolicyOrderingTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -using CPS.ComplexCases.Common.Resilience; -using CPS.ComplexCases.NetApp.Extensions; -using Microsoft.Extensions.DependencyInjection; - -namespace CPS.ComplexCases.NetApp.Tests.Unit.Extensions; - -// Reproduces the registration ordering that previously threw ObjectDisposedException: -// NetAppHttpClient and NetAppS3HttpClient each get their own policy holder, so the second -// client to run must resolve against its own live provider, not a retained, disposed scope. -public class ResiliencePolicyOrderingTests -{ - private static ServiceProvider BuildProvider() => - new ServiceCollection().AddLogging().BuildServiceProvider(); - - [Fact] - public void SecondClientResolvesPolicy_AfterFirstClientScopeDisposed() - { - var firstClient = new SharedResiliencePolicy(IServiceCollectionExtension.GetResiliencePolicy); - var secondClient = new SharedResiliencePolicy(IServiceCollectionExtension.GetResiliencePolicy); - - var firstProvider = BuildProvider(); - var firstPolicy = firstClient.GetPolicy(firstProvider); - - // The first client's handler scope rotates and is disposed before the second client first runs. - firstProvider.Dispose(); - - using var secondProvider = BuildProvider(); - var exception = Record.Exception(() => secondClient.GetPolicy(secondProvider)); - - Assert.Null(exception); - Assert.NotNull(firstPolicy); - } -} diff --git a/backend/CPS.ComplexCases.NetApp.Tests/Unit/Factories/S3ClientFactoryTests.cs b/backend/CPS.ComplexCases.NetApp.Tests/Unit/Factories/S3ClientFactoryTests.cs index 625d06b5b..915a21d88 100644 --- a/backend/CPS.ComplexCases.NetApp.Tests/Unit/Factories/S3ClientFactoryTests.cs +++ b/backend/CPS.ComplexCases.NetApp.Tests/Unit/Factories/S3ClientFactoryTests.cs @@ -49,7 +49,7 @@ public S3ClientFactoryTests() _telemetryHandlerMock = new Mock(); _netAppCertFactoryMock = new Mock(); - var testCert = new X509Certificate2([]); + var testCert = GenerateSelfSignedCertificate("CN=TestCA"); var testCertCollection = new X509Certificate2Collection { testCert }; _netAppCertFactoryMock diff --git a/backend/CPS.ComplexCases.NetApp.WireMock/CPS.ComplexCases.NetApp.WireMock.csproj b/backend/CPS.ComplexCases.NetApp.WireMock/CPS.ComplexCases.NetApp.WireMock.csproj index baa3822e2..0f4d05da0 100644 --- a/backend/CPS.ComplexCases.NetApp.WireMock/CPS.ComplexCases.NetApp.WireMock.csproj +++ b/backend/CPS.ComplexCases.NetApp.WireMock/CPS.ComplexCases.NetApp.WireMock.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -10,13 +10,15 @@ - - - - - - + + + + + + + + diff --git a/backend/CPS.ComplexCases.NetApp/CPS.ComplexCases.NetApp.csproj b/backend/CPS.ComplexCases.NetApp/CPS.ComplexCases.NetApp.csproj index f0dfe2b1b..404320b2c 100644 --- a/backend/CPS.ComplexCases.NetApp/CPS.ComplexCases.NetApp.csproj +++ b/backend/CPS.ComplexCases.NetApp/CPS.ComplexCases.NetApp.csproj @@ -2,7 +2,7 @@ Library - net8.0 + net10.0 enable enable true @@ -16,8 +16,7 @@ - - + diff --git a/backend/CPS.ComplexCases.NetApp/Client/NetAppClient.cs b/backend/CPS.ComplexCases.NetApp/Client/NetAppClient.cs index e8bba7777..9cf589a44 100644 --- a/backend/CPS.ComplexCases.NetApp/Client/NetAppClient.cs +++ b/backend/CPS.ComplexCases.NetApp/Client/NetAppClient.cs @@ -13,7 +13,7 @@ using CPS.ComplexCases.NetApp.Models.Dto; using CPS.ComplexCases.NetApp.Wrappers; using Polly; -using Polly.Contrib.WaitAndRetry; +using Polly.Retry; namespace CPS.ComplexCases.NetApp.Client; @@ -367,10 +367,10 @@ private async Task UploadObjectCoreAsync(UploadObjectArg arg) public async Task UploadPartAsync(UploadPartArg arg) { - var retryPolicy = GetUploadPartRetryPolicy(arg.PartNumber, arg.ObjectKey); + var pipeline = GetUploadPartRetryPolicy(arg.PartNumber, arg.ObjectKey); try { - return await retryPolicy.ExecuteAsync(async () => + return await pipeline.ExecuteAsync(async ct => { // Re-resolve the S3 client on every attempt so that a credential // rotation triggered by a sibling task or another environment is picked up. @@ -388,7 +388,7 @@ private async Task UploadObjectCoreAsync(UploadObjectArg arg) DisablePayloadSigning = true }; return await s3Client.UploadPartAsync(request); - }); + }, CancellationToken.None); } catch (AmazonS3Exception ex) when (ex.ErrorCode == S3ErrorCodes.AccessDenied) { @@ -406,10 +406,10 @@ private async Task UploadObjectCoreAsync(UploadObjectArg arg) public async Task CompleteMultipartUploadAsync(CompleteMultipartUploadArg arg, CancellationToken cancellationToken = default) { - var retryPolicy = GetCompleteMultipartUploadRetryPolicy(arg.UploadId, arg.ObjectKey); + var pipeline = GetCompleteMultipartUploadRetryPolicy(arg.UploadId, arg.ObjectKey); try { - return await retryPolicy.ExecuteAsync(async ct => + return await pipeline.ExecuteAsync(async ct => { var s3Client = await _s3ClientFactory.GetS3ClientAsync(arg.BearerToken); return await s3Client.CompleteMultipartUploadAsync( @@ -984,76 +984,92 @@ private async Task DoesFolderExistInParentListingAsync(string bucketName, return false; } - private Polly.Retry.AsyncRetryPolicy GetDeleteFileRetryPolicy(string objectKey, - string bucketName) + private ResiliencePipeline GetDeleteFileRetryPolicy(string objectKey, string bucketName) { - return Policy - .HandleResult(r => r.HttpStatusCode != HttpStatusCode.NoContent) - .WaitAndRetryAsync( - Backoff.DecorrelatedJitterBackoffV2( - medianFirstRetryDelay: TimeSpan.FromSeconds(1), - retryCount: 3), - onRetry: (outcome, timespan, retryCount, context) => + return new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 3, + Delay = TimeSpan.FromSeconds(1), + BackoffType = DelayBackoffType.Exponential, + UseJitter = true, + ShouldHandle = static args => + ValueTask.FromResult(args.Outcome.Result?.HttpStatusCode != HttpStatusCode.NoContent), + OnRetry = args => { _logger.LogWarning( "Delete object retry attempt {RetryCount} for key {ObjectKey} in bucket {BucketName}. Status: {StatusCode}. Waiting {DelayMs}ms before next retry.", - retryCount, + args.AttemptNumber + 1, objectKey, bucketName, - outcome.Result?.HttpStatusCode, - timespan.TotalMilliseconds); - }); + args.Outcome.Result?.HttpStatusCode, + args.RetryDelay.TotalMilliseconds); + return ValueTask.CompletedTask; + } + }) + .Build(); } - private Polly.Retry.AsyncRetryPolicy GetUploadPartRetryPolicy(int partNumber, string objectKey) + private ResiliencePipeline GetUploadPartRetryPolicy(int partNumber, string objectKey) { // Retry when NetApp rejects the access key because credentials were rotated // by a concurrently running part upload or another environment sharing the same Key Vault. // On retry, InvalidateClientAsync + GetS3ClientAsync will force-regenerate fresh credentials. - return Policy - .HandleResult(r => false) - .Or(IsCredentialError) - .WaitAndRetryAsync( - retryCount: 2, - sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(retryAttempt * 3), - onRetryAsync: async (outcome, timespan, retryCount, context) => + return new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 2, + Delay = TimeSpan.FromSeconds(3), + BackoffType = DelayBackoffType.Linear, + ShouldHandle = new PredicateBuilder() + .Handle(IsCredentialError), + OnRetry = async args => { - _logger.LogWarning(outcome.Exception, + _logger.LogWarning(args.Outcome.Exception, "Credential error uploading part {PartNumber} for {ObjectKey} - credentials likely rotated mid-transfer (StatusCode={StatusCode}, ErrorCode={ErrorCode}). Refreshing and retrying (attempt {RetryCount}/2). Waiting {DelayMs}ms.", partNumber, objectKey, - (outcome.Exception as AmazonS3Exception)?.StatusCode, - (outcome.Exception as AmazonS3Exception)?.ErrorCode, - retryCount, timespan.TotalMilliseconds); + (args.Outcome.Exception as AmazonS3Exception)?.StatusCode, + (args.Outcome.Exception as AmazonS3Exception)?.ErrorCode, + args.AttemptNumber + 1, + args.RetryDelay.TotalMilliseconds); // Invalidate the cached client ONLY on retry (not the first attempt) // so that GetS3ClientAsync regenerates credentials on the next call. await _s3ClientFactory.InvalidateClientAsync(); - }); + } + }) + .Build(); } - private Polly.Retry.AsyncRetryPolicy GetCompleteMultipartUploadRetryPolicy(string uploadId, string objectKey) + private ResiliencePipeline GetCompleteMultipartUploadRetryPolicy( + string uploadId, string objectKey) { - return Policy - .Handle(ex => (int)ex.StatusCode >= 500 - || ex.StatusCode == HttpStatusCode.RequestTimeout - || IsCredentialError(ex)) - .WaitAndRetryAsync( - Backoff.DecorrelatedJitterBackoffV2( - medianFirstRetryDelay: TimeSpan.FromSeconds(3), - retryCount: 5), - onRetryAsync: async (exception, timespan, retryCount, context) => + return new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 5, + Delay = TimeSpan.FromSeconds(3), + BackoffType = DelayBackoffType.Exponential, + UseJitter = true, + ShouldHandle = new PredicateBuilder() + .Handle(ex => (int)ex.StatusCode >= 500 + || ex.StatusCode == HttpStatusCode.RequestTimeout + || IsCredentialError(ex)), + OnRetry = async args => { - _logger.LogWarning(exception, + _logger.LogWarning(args.Outcome.Exception, "CompleteMultipartUpload retry attempt {RetryCount} for upload {UploadId} ({ObjectKey}). Waiting {DelayMs}ms.", - retryCount, uploadId, objectKey, timespan.TotalMilliseconds); + args.AttemptNumber + 1, uploadId, objectKey, args.RetryDelay.TotalMilliseconds); // Force credential regeneration on retry so the next attempt // gets fresh keys from NetApp instead of reusing the dead cache. - if (exception is AmazonS3Exception s3Ex && IsCredentialError(s3Ex)) + if (args.Outcome.Exception is AmazonS3Exception s3Ex && IsCredentialError(s3Ex)) { await _s3ClientFactory.InvalidateClientAsync(); } - }); + } + }) + .Build(); } private static bool IsCredentialError(AmazonS3Exception ex) diff --git a/backend/CPS.ComplexCases.NetApp/Extensions/IServiceCollectionExtension.cs b/backend/CPS.ComplexCases.NetApp/Extensions/IServiceCollectionExtension.cs index 945af2697..0a42a5a27 100644 --- a/backend/CPS.ComplexCases.NetApp/Extensions/IServiceCollectionExtension.cs +++ b/backend/CPS.ComplexCases.NetApp/Extensions/IServiceCollectionExtension.cs @@ -1,8 +1,10 @@ +using System.Net; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Azure.Identity; using Azure.Security.KeyVault.Secrets; +using CPS.ComplexCases.Common.Extensions; using CPS.ComplexCases.Common.Handlers; using CPS.ComplexCases.NetApp.Client; using CPS.ComplexCases.NetApp.Factories; @@ -11,7 +13,6 @@ using CPS.ComplexCases.NetApp.Telemetry; using CPS.ComplexCases.NetApp.Wrappers; using CPS.ComplexCases.Common.Resilience; -using Polly; namespace CPS.ComplexCases.NetApp.Extensions; @@ -19,6 +20,7 @@ public static class IServiceCollectionExtension { private const int RetryAttempts = 3; private const int FirstRetryDelaySeconds = 1; + private const int ConcurrencyLimit = 30; // NetApp transfers can legitimately run for minutes and run at lower request volumes, so the // breaker uses a longer sampling window, a lower throughput requirement and a longer break. @@ -64,6 +66,21 @@ public static void AddNetAppClient(this IServiceCollection services, IConfigurat var isDevelopment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development"; + var configureResilience = ResiliencePipelineExtensions.ConfigureStandardResilience( + "CPS.ComplexCases.NetApp.CircuitBreaker", + new HttpResilienceOptions + { + ServiceName = "NetApp", + RetryAttempts = RetryAttempts, + FirstRetryDelay = TimeSpan.FromSeconds(FirstRetryDelaySeconds), + CircuitBreakerFailureThreshold = CircuitBreakerFailureThreshold, + CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(CircuitBreakerSamplingDurationSeconds), + CircuitBreakerMinimumThroughput = CircuitBreakerMinimumThroughput, + CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(CircuitBreakerDurationOfBreakSeconds), + ConcurrencyLimit = ConcurrencyLimit, + AdditionalRetryableStatusCodes = [HttpStatusCode.TooManyRequests], + }); + services.AddHttpClient(client => { var netAppServiceUrl = configuration["NetAppOptions:ClusterUrl"]; @@ -77,7 +94,7 @@ public static void AddNetAppClient(this IServiceCollection services, IConfigurat }) .ConfigurePrimaryHttpMessageHandler(sp => CreateHttpClientHandler(sp, isDevelopment)) .SetHandlerLifetime(TimeSpan.FromMinutes(5)) - .AddResiliencePolicyHandler(GetResiliencePolicy); + .AddResilienceHandler("netapp-resilience", configureResilience); services.AddHttpClient(client => { @@ -93,7 +110,7 @@ public static void AddNetAppClient(this IServiceCollection services, IConfigurat .ConfigurePrimaryHttpMessageHandler(sp => CreateHttpClientHandler(sp, isDevelopment) ) .SetHandlerLifetime(TimeSpan.FromMinutes(5)) - .AddResiliencePolicyHandler(GetResiliencePolicy); + .AddResilienceHandler("netapp-s3-resilience", configureResilience); services.AddHttpClient(client => { @@ -108,7 +125,7 @@ public static void AddNetAppClient(this IServiceCollection services, IConfigurat }) .ConfigurePrimaryHttpMessageHandler(sp => CreateHttpClientHandler(sp, isDevelopment)) .SetHandlerLifetime(TimeSpan.FromMinutes(5)) - .AddResiliencePolicyHandler(GetResiliencePolicy); + .AddResilienceHandler("ontap-resilience", configureResilience); services.AddTransient(); } @@ -139,21 +156,4 @@ private static HttpClientHandler CreateHttpClientHandler(IServiceProvider sp, bo "Ensure RootCaCert, IssuingCaCert, and/or IssuingCaCert2 are correctly configured."); } } - - internal static IAsyncPolicy GetResiliencePolicy(ILoggerFactory loggerFactory) - { - var logger = loggerFactory.CreateLogger("CPS.ComplexCases.NetApp.CircuitBreaker"); - - return HttpResiliencePolicyFactory.CreateRateLimitedResiliencePolicy(logger, new HttpResilienceOptions - { - ServiceName = "NetApp", - RetryAttempts = RetryAttempts, - FirstRetryDelay = TimeSpan.FromSeconds(FirstRetryDelaySeconds), - CircuitBreakerFailureThreshold = CircuitBreakerFailureThreshold, - CircuitBreakerSamplingDuration = TimeSpan.FromSeconds(CircuitBreakerSamplingDurationSeconds), - CircuitBreakerMinimumThroughput = CircuitBreakerMinimumThroughput, - CircuitBreakerDurationOfBreak = TimeSpan.FromSeconds(CircuitBreakerDurationOfBreakSeconds), - BulkheadMaxParallelization = 30, - }); - } } \ No newline at end of file diff --git a/backend/CPS.ComplexCases.NetApp/Factories/NetAppCertFactory.cs b/backend/CPS.ComplexCases.NetApp/Factories/NetAppCertFactory.cs index d469d2217..382eda334 100644 --- a/backend/CPS.ComplexCases.NetApp/Factories/NetAppCertFactory.cs +++ b/backend/CPS.ComplexCases.NetApp/Factories/NetAppCertFactory.cs @@ -28,7 +28,7 @@ public X509Certificate2Collection GetTrustedCaCertificates() try { var rootCaBytes = Convert.FromBase64String(rootCaBase64); - var rootCaCert = new X509Certificate2(rootCaBytes); + var rootCaCert = X509CertificateLoader.LoadCertificate(rootCaBytes); _trustedCaCertificates.Add(rootCaCert); } catch (Exception ex) @@ -46,7 +46,7 @@ public X509Certificate2Collection GetTrustedCaCertificates() try { var issuingCaBytes = Convert.FromBase64String(issuingCaBase64); - var issuingCaCert = new X509Certificate2(issuingCaBytes); + var issuingCaCert = X509CertificateLoader.LoadCertificate(issuingCaBytes); _trustedCaCertificates.Add(issuingCaCert); } catch (Exception ex) @@ -64,7 +64,7 @@ public X509Certificate2Collection GetTrustedCaCertificates() try { var issuingCa2Bytes = Convert.FromBase64String(issuingCa2Base64); - var issuingCa2Cert = new X509Certificate2(issuingCa2Bytes); + var issuingCa2Cert = X509CertificateLoader.LoadCertificate(issuingCa2Bytes); _trustedCaCertificates.Add(issuingCa2Cert); } catch (Exception ex) diff --git a/backend/CPS.ComplexCases.NetApp/Factories/S3ClientFactory.cs b/backend/CPS.ComplexCases.NetApp/Factories/S3ClientFactory.cs index 56c99c7c4..b52b5c294 100644 --- a/backend/CPS.ComplexCases.NetApp/Factories/S3ClientFactory.cs +++ b/backend/CPS.ComplexCases.NetApp/Factories/S3ClientFactory.cs @@ -1,5 +1,4 @@ using System.IdentityModel.Tokens.Jwt; -using System.Net; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Amazon; @@ -152,9 +151,7 @@ private async Task CreateS3Client(string bearerToken, bool forceRegen } else if (isDevelopment) { - // In Development, bypass all SSL validation - ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; - + // In Development, bypass all SSL validation via HttpClientHandler var handler = new HttpClientHandler { ServerCertificateCustomValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true @@ -169,8 +166,6 @@ private async Task CreateS3Client(string bearerToken, bool forceRegen "Please ensure that the Root CA and Issuing CA certificates are correctly configured in Key Vault."); } - ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls13; - var s3Client = new AmazonS3Client(credentials, s3Config); s3Client.BeforeRequestEvent += (sender, args) => diff --git a/backend/CPS.ComplexCases.WireMock.Core/CPS.ComplexCases.WireMock.Core.csproj b/backend/CPS.ComplexCases.WireMock.Core/CPS.ComplexCases.WireMock.Core.csproj index 100728de8..790f7d6a8 100644 --- a/backend/CPS.ComplexCases.WireMock.Core/CPS.ComplexCases.WireMock.Core.csproj +++ b/backend/CPS.ComplexCases.WireMock.Core/CPS.ComplexCases.WireMock.Core.csproj @@ -1,7 +1,7 @@  - net8.0 + net10.0 enable enable true @@ -9,6 +9,8 @@ + + \ No newline at end of file diff --git a/backend/CPS.ComplexCases.WireMock/CPS.ComplexCases.WireMock.csproj b/backend/CPS.ComplexCases.WireMock/CPS.ComplexCases.WireMock.csproj index 346dc5dd0..f8bc2852b 100644 --- a/backend/CPS.ComplexCases.WireMock/CPS.ComplexCases.WireMock.csproj +++ b/backend/CPS.ComplexCases.WireMock/CPS.ComplexCases.WireMock.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net10.0 enable enable true @@ -10,6 +10,8 @@ + + - net8.0 + net10.0 enable enable @@ -18,9 +18,11 @@ + + - + diff --git a/devops-pipelines/backend/backend-build-and-deploy.yml b/devops-pipelines/backend/backend-build-and-deploy.yml index d9d90a17c..0ba2ccf87 100644 --- a/devops-pipelines/backend/backend-build-and-deploy.yml +++ b/devops-pipelines/backend/backend-build-and-deploy.yml @@ -25,9 +25,9 @@ variables: - name: buildConfiguration value: "Release" - name: dotNetVersion - value: "8.x" + value: '10.x' - name: efVersion - value: "8.0.*" + value: '10.0.*' - name: asposeLicenceFileName value: "Aspose.Total.NET.lic" diff --git a/devops-pipelines/backend/backend-pr-build-and-test.yml b/devops-pipelines/backend/backend-pr-build-and-test.yml index 59598393e..c1f96d245 100644 --- a/devops-pipelines/backend/backend-pr-build-and-test.yml +++ b/devops-pipelines/backend/backend-pr-build-and-test.yml @@ -23,9 +23,9 @@ variables: - name: buildConfiguration value: "Release" - name: dotNetVersion - value: "8.x" + value: '10.x' - name: efVersion - value: "8.0.*" + value: '10.0.*' - name: testResultsDirectory value: "$(Agent.TempDirectory)/TestResults" diff --git a/devops-pipelines/backend/backend-pr-integration-tests.yml b/devops-pipelines/backend/backend-pr-integration-tests.yml index c81ff5097..e23bd5e10 100644 --- a/devops-pipelines/backend/backend-pr-integration-tests.yml +++ b/devops-pipelines/backend/backend-pr-integration-tests.yml @@ -17,9 +17,9 @@ variables: - name: buildConfiguration value: "Release" - name: dotNetVersion - value: "8.x" + value: "10.x" - name: efVersion - value: "8.0.*" + value: "10.0.*" - name: testResultsDirectory value: "$(Agent.TempDirectory)/TestResults" diff --git a/devops-pipelines/templates/dotnet-build-steps.yml b/devops-pipelines/templates/dotnet-build-steps.yml index cd742ee24..c1d31b357 100644 --- a/devops-pipelines/templates/dotnet-build-steps.yml +++ b/devops-pipelines/templates/dotnet-build-steps.yml @@ -2,7 +2,7 @@ parameters: projectPath: "" projectName: "" buildConfiguration: "Release" - dotNetVersion: "8.x" + dotNetVersion: "10.x" useLocalNuGet: false nugetArtifact: "localnuget" nugetArtifactDownloadPath: "$(Build.SourcesDirectory)/localnuget" diff --git a/devops-pipelines/templates/fa-config-steps.yml b/devops-pipelines/templates/fa-config-steps.yml index afbda9a5d..72f419345 100644 --- a/devops-pipelines/templates/fa-config-steps.yml +++ b/devops-pipelines/templates/fa-config-steps.yml @@ -316,7 +316,7 @@ steps: }, { "name": "DOTNET_FRAMEWORK_VERSION", - "value": "v8.0", + "value": "v10.0", "slotSetting": false }, { diff --git a/devops-pipelines/templates/fa-deploy-steps.yml b/devops-pipelines/templates/fa-deploy-steps.yml index 15fd20ecb..89f295aed 100644 --- a/devops-pipelines/templates/fa-deploy-steps.yml +++ b/devops-pipelines/templates/fa-deploy-steps.yml @@ -34,7 +34,7 @@ steps: package: "$(Pipeline.Workspace)/${{ parameters.buildArtifactName }}/*.zip" ${{ else }}: package: "$(Pipeline.Workspace)/${{ parameters.pipelineResource}}/${{ parameters.buildArtifactName }}/*.zip" - runtimeStack: "DOTNET-ISOLATED|8.0" + runtimeStack: "DOTNET-ISOLATED|10.0" deploymentMethod: "runFromPackage" deployToSlotOrASE: ${{ parameters.deployToSlot }} slotName: ${{ parameters.slotName }} diff --git a/devops-pipelines/ui/ui-pr-build-and-test.yml b/devops-pipelines/ui/ui-pr-build-and-test.yml index d9ed39125..aa4e86031 100644 --- a/devops-pipelines/ui/ui-pr-build-and-test.yml +++ b/devops-pipelines/ui/ui-pr-build-and-test.yml @@ -70,7 +70,7 @@ stages: displayName: "Use .NET SDK to publish Code Coverage" inputs: packageType: "sdk" - version: "8.x" + version: "10.x" - script: | export PATH="$PATH:$HOME/.dotnet/tools" diff --git a/scripts/build-and-test-backend-local.ps1 b/scripts/build-and-test-backend-local.ps1 index f417106aa..2b5ab6258 100644 --- a/scripts/build-and-test-backend-local.ps1 +++ b/scripts/build-and-test-backend-local.ps1 @@ -87,7 +87,7 @@ try { } catch { Write-Host "ERROR .NET SDK not found" -ForegroundColor Red - Write-Host "Please install .NET 8 SDK from https://dotnet.microsoft.com/download" -ForegroundColor Yellow + Write-Host "Please install .NET 10 SDK from https://dotnet.microsoft.com/download" -ForegroundColor Yellow exit 1 } diff --git a/scripts/build-and-test-local.ps1 b/scripts/build-and-test-local.ps1 index 8142c8f35..d361b9236 100644 --- a/scripts/build-and-test-local.ps1 +++ b/scripts/build-and-test-local.ps1 @@ -118,7 +118,7 @@ try { } catch { Write-Host "ERROR .NET SDK not found" -ForegroundColor Red - Write-Host "Please install .NET 8 SDK from https://dotnet.microsoft.com/download" -ForegroundColor Yellow + Write-Host "Please install .NET 10 SDK from https://dotnet.microsoft.com/download" -ForegroundColor Yellow exit 1 }