From 20d6cd7f0e7085ef0e9d6474c9f8f6c260478da3 Mon Sep 17 00:00:00 2001 From: Lilach Date: Wed, 23 Sep 2026 14:48:19 +0100 Subject: [PATCH 1/6] Task/fct2 22065 revise cicd for new branching strategy (#583) * filtering status telemetry (#581) * filtering status telemetry * fix nuget * pr comment fixes * add version to dotnet build * deploy envs conditionally, require manualvalidation only for prod * story/FCT2-20672 : Text content update for the unauthorised page (#584) * add separate pipeline yaml for development trigger * unblock E2E Tests in Dev * test release versioning in dev * test release versioning in dev * use var instead of parameter for buildConfiguration * remove temorary test lines * task/FCT2-21994 backend test pipeline - rm redundant build job (#585) * egress delete check all files successful (#586) * Fct2 21924 bulk delete unknown files (#587) * deletion of unexisting files * deletion of existing files * update parameter empty string value option * trigger integration tests only for debelopment * rm unused parameters, fix stage dependency * fix stage dependency --------- Co-authored-by: Rhys Bridges Co-authored-by: Renjith Abby --- ...omplexCases.API.HttpTelemetry.Tests.csproj | 26 ++++ ...althCheckTelemetryHostRegistrationTests.cs | 97 ++++++++++++ .../CPS.ComplexCases.API.HttpTelemetry.csproj | 11 +- .../HealthCheckTelemetryHostRegistration.cs | 47 ++++++ .../Startup.cs | 2 + .../Attributes/HttpTelemetryAttribute.cs | 2 +- .../CPS.ComplexCases.API.csproj | 8 +- .../HealthCheckTelemetryFilterTests.cs | 33 +++- .../Telemetry/HealthCheckTelemetryFilter.cs | 24 ++- .../Unit/EgressStorageClientTests.cs | 112 +++++++++++++- .../Client/EgressStorageClient.cs | 45 +++++- .../Models/Response/DeleteFilesResponse.cs | 16 +- .../Unit/Durable/Activity/DeleteFilesTests.cs | 26 ++++ .../Activity/UpdateActivityLogTests.cs | 63 ++++++++ .../Attributes/HttpTelemetryExtension.cs | 5 + .../CPS.ComplexCases.FileTransfer.API.csproj | 3 +- .../Durable/Activity/DeleteFiles.cs | 17 ++- .../Durable/Activity/UpdateActivityLog.cs | 11 +- backend/CPS.ComplexCases.sln | 14 ++ devops-pipelines/api-postman-test.yml | 4 +- .../backend/backend-build-and-deploy.yml | 49 +++--- .../backend/backend-dev-build-and-deploy.yml | 142 ++++++++++++++++++ .../backend/backend-pr-build-and-test.yml | 74 +-------- .../backend/backend-pr-integration-tests.yml | 2 +- devops-pipelines/backend/db-rollback.yml | 3 - devops-pipelines/cleanup-testfiles.yml | 4 +- .../templates/backend-deploy-stages.yml | 32 +--- .../templates/dotnet-build-steps.yml | 67 +++++++-- .../templates/e2e-playwright-test-job.yml | 4 +- .../templates/ui-build-and-deploy-stages.yml | 22 --- devops-pipelines/ui/ui-build-and-deploy.yml | 31 ++-- .../ui/ui-dev-build-and-deploy.yml | 14 ++ devops-pipelines/ui/ui-pr-build-and-test.yml | 1 + .../integration-tests/case-search.spec.ts | 14 +- ui-spa/src/components/unauthorised/index.tsx | 8 +- 35 files changed, 800 insertions(+), 233 deletions(-) create mode 100644 backend/CPS.ComplexCases.API.HttpTelemetry.Tests/CPS.ComplexCases.API.HttpTelemetry.Tests.csproj create mode 100644 backend/CPS.ComplexCases.API.HttpTelemetry.Tests/HealthCheckTelemetryHostRegistrationTests.cs create mode 100644 backend/CPS.ComplexCases.API.HttpTelemetry/HealthCheckTelemetryHostRegistration.cs create mode 100644 backend/CPS.ComplexCases.FileTransfer.API/Attributes/HttpTelemetryExtension.cs create mode 100644 devops-pipelines/backend/backend-dev-build-and-deploy.yml create mode 100644 devops-pipelines/ui/ui-dev-build-and-deploy.yml diff --git a/backend/CPS.ComplexCases.API.HttpTelemetry.Tests/CPS.ComplexCases.API.HttpTelemetry.Tests.csproj b/backend/CPS.ComplexCases.API.HttpTelemetry.Tests/CPS.ComplexCases.API.HttpTelemetry.Tests.csproj new file mode 100644 index 000000000..c3a6c5207 --- /dev/null +++ b/backend/CPS.ComplexCases.API.HttpTelemetry.Tests/CPS.ComplexCases.API.HttpTelemetry.Tests.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + + + + + + + + + + + + diff --git a/backend/CPS.ComplexCases.API.HttpTelemetry.Tests/HealthCheckTelemetryHostRegistrationTests.cs b/backend/CPS.ComplexCases.API.HttpTelemetry.Tests/HealthCheckTelemetryHostRegistrationTests.cs new file mode 100644 index 000000000..7c4ccb256 --- /dev/null +++ b/backend/CPS.ComplexCases.API.HttpTelemetry.Tests/HealthCheckTelemetryHostRegistrationTests.cs @@ -0,0 +1,97 @@ +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.DependencyInjection; + +namespace CPS.ComplexCases.API.HttpTelemetry.Tests; + +public class HealthCheckTelemetryHostRegistrationTests +{ + [Fact] + public void Register_WrapsTelemetryConfiguration_AndDropsStatusHostTelemetry() + { + var channel = new StubChannel(); + var services = new ServiceCollection(); + services.AddSingleton(_ => new TelemetryConfiguration { TelemetryChannel = channel }); + + HealthCheckTelemetryHostRegistration.Register(services); + + var config = services.BuildServiceProvider().GetRequiredService(); + var client = new TelemetryClient(config); + + var statusRequest = new RequestTelemetry + { + Name = "Status", + Url = new Uri("https://example.com/api/status") + }; + var invokeDependency = new DependencyTelemetry + { + Name = "Invoke", + Type = "InProc" + }; + invokeDependency.Context.Operation.Name = "Functions.Status"; + + client.TrackRequest(statusRequest); + client.Track(invokeDependency); + client.TrackRequest(new RequestTelemetry + { + Name = "GetCase", + Url = new Uri("https://example.com/api/v1/cases/1") + }); + client.Flush(); + + var remaining = Assert.Single(channel.Items); + var remainingRequest = Assert.IsType(remaining); + Assert.Equal("GetCase", remainingRequest.Name); + } + + [Fact] + public void Register_AddsTelemetryModule_WhenConfigurationIsNotRegistered() + { + var services = new ServiceCollection(); + + HealthCheckTelemetryHostRegistration.Register(services); + + Assert.Contains(services, descriptor => + descriptor.ServiceType == typeof(ITelemetryModule) + && descriptor.ImplementationType == typeof(HealthCheckTelemetryModule)); + } + + [Fact] + public void Module_Initialize_DropsStatusRequestsAfterBuild() + { + var channel = new StubChannel(); + var config = new TelemetryConfiguration { TelemetryChannel = channel }; + + new HealthCheckTelemetryModule().Initialize(config); + config.TelemetryProcessorChainBuilder.Build(); + + var client = new TelemetryClient(config); + client.TrackRequest(new RequestTelemetry + { + Name = "Status", + Url = new Uri("https://example.com/api/status") + }); + client.TrackRequest(new RequestTelemetry + { + Name = "GetCase", + Url = new Uri("https://example.com/api/v1/cases/1") + }); + client.Flush(); + + var remaining = Assert.Single(channel.Items); + var remainingRequest = Assert.IsType(remaining); + Assert.Equal("GetCase", remainingRequest.Name); + } + + private sealed class StubChannel : ITelemetryChannel + { + public List Items { get; } = []; + public bool? DeveloperMode { get; set; } + public string? EndpointAddress { get; set; } + public void Send(ITelemetry item) => Items.Add(item); + public void Flush() { } + public void Dispose() { } + } +} diff --git a/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj b/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj index 7f48d1917..c55658fde 100644 --- a/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj +++ b/backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj @@ -2,11 +2,12 @@ - 1.0.41 + 1.0.42 net8.0;net10.0 enable enable + $(DefineConstants);HOST_TELEMETRY True Brian Dunnington @@ -26,5 +27,11 @@ + + + + <_Parameter1>CPS.ComplexCases.API.HttpTelemetry.Tests + + - \ No newline at end of file + diff --git a/backend/CPS.ComplexCases.API.HttpTelemetry/HealthCheckTelemetryHostRegistration.cs b/backend/CPS.ComplexCases.API.HttpTelemetry/HealthCheckTelemetryHostRegistration.cs new file mode 100644 index 000000000..48a4c5589 --- /dev/null +++ b/backend/CPS.ComplexCases.API.HttpTelemetry/HealthCheckTelemetryHostRegistration.cs @@ -0,0 +1,47 @@ +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.DependencyInjection; + +namespace CPS.ComplexCases.API.HttpTelemetry; + +internal static class HealthCheckTelemetryHostRegistration +{ + public static void Register(IServiceCollection services) + { + var configDescriptor = services.FirstOrDefault(descriptor => descriptor.ServiceType == typeof(TelemetryConfiguration)); + if (configDescriptor?.ImplementationFactory is not null) + { + var implFactory = configDescriptor.ImplementationFactory; + services.Remove(configDescriptor); + services.AddSingleton(provider => + { + var config = (TelemetryConfiguration)implFactory(provider); + Attach(config); + return config; + }); + return; + } + + if (configDescriptor?.ImplementationInstance is TelemetryConfiguration instance) + { + Attach(instance); + return; + } + + services.AddSingleton(); + } + + internal static void Attach(TelemetryConfiguration config) + { + config.TelemetryProcessorChainBuilder + .Use(next => new HealthCheckTelemetryFilter(next)) + .Build(); + } +} + +internal sealed class HealthCheckTelemetryModule : ITelemetryModule +{ + public void Initialize(TelemetryConfiguration configuration) + { + configuration.TelemetryProcessorChainBuilder.Use(next => new HealthCheckTelemetryFilter(next)); + } +} diff --git a/backend/CPS.ComplexCases.API.HttpTelemetry/Startup.cs b/backend/CPS.ComplexCases.API.HttpTelemetry/Startup.cs index 77bbc3420..bf5339906 100644 --- a/backend/CPS.ComplexCases.API.HttpTelemetry/Startup.cs +++ b/backend/CPS.ComplexCases.API.HttpTelemetry/Startup.cs @@ -24,6 +24,8 @@ public void Configure(WebJobsBuilderContext context, IWebJobsBuilder builder) builder.AddExtension(); builder.Services.AddSingleton(); builder.Services.AddHttpContextAccessor(); + // Runs in the Functions host process. The worker ITelemetryProcessor cannot see host Request/Invoke telemetry. + HealthCheckTelemetryHostRegistration.Register(builder.Services); } } diff --git a/backend/CPS.ComplexCases.API/Attributes/HttpTelemetryAttribute.cs b/backend/CPS.ComplexCases.API/Attributes/HttpTelemetryAttribute.cs index 9feab45ad..872753133 100644 --- a/backend/CPS.ComplexCases.API/Attributes/HttpTelemetryAttribute.cs +++ b/backend/CPS.ComplexCases.API/Attributes/HttpTelemetryAttribute.cs @@ -1,6 +1,6 @@ using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; -[assembly: ExtensionInformation("CPS.ComplexCases.API.HttpTelemetry", "1.0.41")] +[assembly: ExtensionInformation("CPS.ComplexCases.API.HttpTelemetry", "1.0.42")] namespace CPS.ComplexCases.API.Attributes { [AttributeUsage(AttributeTargets.Parameter)] diff --git a/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj b/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj index 63daceccd..2f9727258 100644 --- a/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj +++ b/backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj @@ -16,8 +16,8 @@ - - + + @@ -33,7 +33,7 @@ - + @@ -68,4 +68,4 @@ - \ No newline at end of file + diff --git a/backend/CPS.ComplexCases.Common.Tests/Unit/Telemetry/HealthCheckTelemetryFilterTests.cs b/backend/CPS.ComplexCases.Common.Tests/Unit/Telemetry/HealthCheckTelemetryFilterTests.cs index 0107319ab..9d7e3f8d0 100644 --- a/backend/CPS.ComplexCases.Common.Tests/Unit/Telemetry/HealthCheckTelemetryFilterTests.cs +++ b/backend/CPS.ComplexCases.Common.Tests/Unit/Telemetry/HealthCheckTelemetryFilterTests.cs @@ -16,11 +16,15 @@ public HealthCheckTelemetryFilterTests() _sut = new HealthCheckTelemetryFilter(_next.Object); } - [Fact] - public void Process_FiltersRequest_WhenFunctionNameIsStatus() + [Theory] + [InlineData("Status")] + [InlineData("Functions.Status")] + [InlineData("functions.status")] + [InlineData(" Functions.Status ")] + public void Process_FiltersRequest_WhenFunctionNameIsStatus(string functionName) { var request = new RequestTelemetry { Name = "GetCase" }; - request.Properties["AzureFunctions_FunctionName"] = "Status"; + request.Properties["AzureFunctions_FunctionName"] = functionName; AssertFiltered(request); } @@ -58,6 +62,7 @@ public void Process_FiltersRequest_WhenUrlIsPeriodicHealthEndpoint(string url) [Theory] [InlineData("Status")] + [InlineData("Functions.Status")] [InlineData("GET /api/status")] [InlineData("GET /api/health")] public void Process_FiltersRequest_WhenNameIsPeriodicHealthEndpoint(string name) @@ -83,6 +88,8 @@ public void Process_FiltersDependency_WhenDataIsPeriodicHealthEndpoint(string da } [Theory] + [InlineData("Status")] + [InlineData("Functions.Status")] [InlineData("GET /api/status")] [InlineData("GET /api/health")] [InlineData("/api/status")] @@ -109,6 +116,7 @@ public void Process_FiltersDependency_WhenTargetIsPeriodicHealthEndpoint() [Theory] [InlineData("Status")] + [InlineData("Functions.Status")] [InlineData("GET /api/status")] [InlineData("GET /api/health")] public void Process_FiltersDependency_WhenOperationNameIsPeriodicHealthEndpoint(string operationName) @@ -139,6 +147,15 @@ public void Process_DoesNotFilterRequest_WhenPathIsTransferStatus() AssertPassedThrough(request); } + [Fact] + public void Process_DoesNotFilterRequest_WhenFunctionNameIsPrefixedTransferStatus() + { + var request = new RequestTelemetry { Name = "GetTransferStatus" }; + request.Properties["AzureFunctions_FunctionName"] = "Functions.GetTransferStatus"; + + AssertPassedThrough(request); + } + [Fact] public void Process_DoesNotFilterDependency_WhenDataIsTransferStatus() { @@ -154,6 +171,16 @@ public void Process_DoesNotFilterDependency_WhenDataIsTransferStatus() AssertPassedThrough(dependency); } + [Fact] + public void Process_DoesNotFilterDependency_WhenNameIsInvokeWithoutHealthOperation() + { + AssertPassedThrough(new DependencyTelemetry + { + Name = "Invoke", + Type = "InProc" + }); + } + [Fact] public void Process_DoesNotFilterDependency_WhenUnrelatedToHealthEndpoints() { diff --git a/backend/CPS.ComplexCases.Common/Telemetry/HealthCheckTelemetryFilter.cs b/backend/CPS.ComplexCases.Common/Telemetry/HealthCheckTelemetryFilter.cs index bd957e567..ccfa47d73 100644 --- a/backend/CPS.ComplexCases.Common/Telemetry/HealthCheckTelemetryFilter.cs +++ b/backend/CPS.ComplexCases.Common/Telemetry/HealthCheckTelemetryFilter.cs @@ -2,7 +2,11 @@ using Microsoft.ApplicationInsights.DataContracts; using Microsoft.ApplicationInsights.Extensibility; +#if HOST_TELEMETRY +namespace CPS.ComplexCases.API.HttpTelemetry; +#else namespace CPS.ComplexCases.Common.Telemetry; +#endif public class HealthCheckTelemetryFilter(ITelemetryProcessor next) : ITelemetryProcessor { @@ -13,6 +17,7 @@ public class HealthCheckTelemetryFilter(ITelemetryProcessor next) : ITelemetryPr ]; private const string StatusFunctionName = "Status"; + private const string FunctionsPrefix = "Functions."; private readonly ITelemetryProcessor _next = next; @@ -58,15 +63,28 @@ private static bool ShouldFilter(ITelemetry item) if (item is DependencyTelemetry dependency) { return IsPeriodicHealthProbeValue(dependency.Data) - || IsPeriodicHealthProbeValue(dependency.Name) + || IsPeriodicHealthOperation(dependency.Name) || IsPeriodicHealthProbeValue(dependency.Target); } return false; } - private static bool IsPeriodicHealthFunctionName(string? name) => - string.Equals(name, StatusFunctionName, StringComparison.OrdinalIgnoreCase); + private static bool IsPeriodicHealthFunctionName(string? name) + { + if (string.IsNullOrWhiteSpace(name)) + { + return false; + } + + var trimmed = name.Trim(); + if (trimmed.StartsWith(FunctionsPrefix, StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[FunctionsPrefix.Length..]; + } + + return string.Equals(trimmed, StatusFunctionName, StringComparison.OrdinalIgnoreCase); + } private static bool IsPeriodicHealthOperation(string? name) => IsPeriodicHealthFunctionName(name) || IsPeriodicHealthProbeValue(name); diff --git a/backend/CPS.ComplexCases.Egress.Tests/Unit/EgressStorageClientTests.cs b/backend/CPS.ComplexCases.Egress.Tests/Unit/EgressStorageClientTests.cs index c42ff5576..e7c0f05d1 100644 --- a/backend/CPS.ComplexCases.Egress.Tests/Unit/EgressStorageClientTests.cs +++ b/backend/CPS.ComplexCases.Egress.Tests/Unit/EgressStorageClientTests.cs @@ -367,13 +367,12 @@ public async Task DeleteFilesAsync_WithCodeZeroAndNullFileId_PopulatesDeletedFil var result = await _client.DeleteFilesAsync(filesToDelete, workspaceId); Assert.True(result.AllSuccessful); - Assert.Equal(2, result.DeletedFiles!.Count); Assert.Equal(["file1.txt", "file2.txt"], result.DeletedFiles); Assert.Empty(result.FailedFiles!); } [Fact] - public async Task DeleteFilesAsync_WithCodeZeroAndNullFileIdAndFilename_UsesDeletedFallback() + public async Task DeleteFilesAsync_WithCodeZeroAndNullFileIdAndFilename_UsesRequestedFileIds() { var workspaceId = _fixture.Create(); var token = _fixture.Create(); @@ -397,7 +396,8 @@ public async Task DeleteFilesAsync_WithCodeZeroAndNullFileIdAndFilename_UsesDele var result = await _client.DeleteFilesAsync(filesToDelete, workspaceId); - Assert.Equal(["deleted"], result.DeletedFiles); + Assert.True(result.AllSuccessful); + Assert.Equal(["file-1"], result.DeletedFiles); Assert.Empty(result.FailedFiles!); } @@ -741,6 +741,112 @@ public async Task DeleteFilesAsync_WhenEgressOmitsFiles_ReturnsNotAllSuccessful( Assert.Empty(result.FailedFiles!); } + [Fact] + public async Task DeleteFilesAsync_WhenEgressReturnsEmptyResultsForUnknownId_DoesNotTreatAsDeleted() + { + var workspaceId = _fixture.Create(); + var token = _fixture.Create(); + var filesToDelete = new List + { + new() { Path = "folder/missing.txt", FileId = "000000000000000000000000" } + }; + + SetupTokenRequest(token); + SetupDeleteFilesRequest(workspaceId, token); + SetupHttpMockResponses( + ("token", new GetWorkspaceTokenResponse { Token = token }), + ("delete", new { all_successful = true, results = Array.Empty() })); + + var result = await _client.DeleteFilesAsync(filesToDelete, workspaceId); + + Assert.False(result.AllSuccessful); + Assert.Empty(result.DeletedFiles!); + Assert.Empty(result.FailedFiles!); + } + + [Fact] + public async Task DeleteFilesAsync_WhenEgressReturnsEmptyFilesForUnknownId_DoesNotTreatAsDeleted() + { + var workspaceId = _fixture.Create(); + var token = _fixture.Create(); + var filesToDelete = new List + { + new() { Path = "folder/missing.txt", FileId = "000000000000000000000000" } + }; + + SetupTokenRequest(token); + SetupDeleteFilesRequest(workspaceId, token); + SetupHttpMockResponses( + ("token", new GetWorkspaceTokenResponse { Token = token }), + ("delete", new { all_successful = true, files = Array.Empty() })); + + var result = await _client.DeleteFilesAsync(filesToDelete, workspaceId); + + Assert.False(result.AllSuccessful); + Assert.Empty(result.DeletedFiles!); + Assert.Empty(result.FailedFiles!); + } + + [Fact] + public async Task DeleteFilesAsync_WhenEgressReturnsFilesWithId_PopulatesDeletedFiles() + { + var workspaceId = _fixture.Create(); + var token = _fixture.Create(); + var filesToDelete = new List + { + new() { Path = "1. ABEs for Transcript/Free_Test_Data_10.5MB_PDF.pdf", FileId = "6a7b09840b11b5e3185286b7" } + }; + + SetupTokenRequest(token); + SetupDeleteFilesRequest(workspaceId, token); + SetupHttpMockResponses( + ("token", new GetWorkspaceTokenResponse { Token = token }), + ("delete", new + { + all_successful = true, + files = new[] + { + new { code = 0, id = "6a7b09840b11b5e3185286b7", filename = "Free_Test_Data_10.5MB_PDF.pdf", is_folder = false } + } + })); + + var result = await _client.DeleteFilesAsync(filesToDelete, workspaceId); + + Assert.True(result.AllSuccessful); + Assert.Equal(["6a7b09840b11b5e3185286b7"], result.DeletedFiles); + Assert.Empty(result.FailedFiles!); + } + + [Fact] + public async Task DeleteFilesAsync_WhenEgressReturnsResultsWithId_PopulatesDeletedFiles() + { + var workspaceId = _fixture.Create(); + var token = _fixture.Create(); + var filesToDelete = new List + { + new() { Path = "4. Served Evidence/1000mb.txt", FileId = "6a7b09840b11b5e3185286b7" } + }; + + SetupTokenRequest(token); + SetupDeleteFilesRequest(workspaceId, token); + SetupHttpMockResponses( + ("token", new GetWorkspaceTokenResponse { Token = token }), + ("delete", new + { + all_successful = true, + results = new[] + { + new { code = 0, id = "6a7b09840b11b5e3185286b7", filename = "1000mb.txt", is_folder = false } + } + })); + + var result = await _client.DeleteFilesAsync(filesToDelete, workspaceId); + + Assert.True(result.AllSuccessful); + Assert.Equal(["6a7b09840b11b5e3185286b7"], result.DeletedFiles); + Assert.Empty(result.FailedFiles!); + } + [Fact] public async Task DeleteFilesAsync_WhenEgressReportsNotAllSuccessful_ReturnsNotAllSuccessful() { diff --git a/backend/CPS.ComplexCases.Egress/Client/EgressStorageClient.cs b/backend/CPS.ComplexCases.Egress/Client/EgressStorageClient.cs index 5475856c3..679673e5f 100644 --- a/backend/CPS.ComplexCases.Egress/Client/EgressStorageClient.cs +++ b/backend/CPS.ComplexCases.Egress/Client/EgressStorageClient.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Text.Json; using CPS.ComplexCases.Common.Extensions; using CPS.ComplexCases.Common.Models.Domain; using CPS.ComplexCases.Common.Models.Domain.Dtos; @@ -272,21 +273,51 @@ public async Task DeleteFilesAsync(List fi FileIds = [.. chunk] }; - var result = await SendRequestAsync(_egressRequestFactory.DeleteFilesRequest(deleteArg, token)); + using var response = await SendRequestAsync(_egressRequestFactory.DeleteFilesRequest(deleteArg, token)); + var responseContent = await response.Content.ReadAsStringAsync(); + var result = JsonSerializer.Deserialize(responseContent) + ?? throw new InvalidOperationException("Deserialization returned null."); allSuccessful &= result.AllSuccessful; - var files = result.Files ?? []; + var files = result.Items ?? []; + if (files.Count == 0) + { + _logger.LogWarning( + "Egress bulk delete returned no per-file confirmations for workspace {WorkspaceId}. AllSuccessful={AllSuccessful}, RequestedCount={RequestedCount}, Response={Response}", + workspaceId, + result.AllSuccessful, + chunk.Length, + responseContent); + } - deletedFiles.AddRange(files - .Where(x => x.Code == 0) - .Select(x => x.FileId ?? x.Filename ?? "deleted")); + var failedResults = files.Where(x => x.Code > 0).ToList(); + var successfulResults = files.Where(x => x.Code == 0).ToList(); - failedFiles.AddRange(files.Where(x => x.Code > 0).Select(x => new FailedFileDeletion + failedFiles.AddRange(failedResults.Select(x => new FailedFileDeletion { - FileId = x.FileId ?? x.Filename ?? string.Empty, + FileId = x.ResolvedFileId ?? string.Empty, Filename = x.Filename ?? string.Empty, Reason = GetDeleteFailureReason(x) })); + + var identifiedDeleted = successfulResults + .Select(x => x.ResolvedFileId) + .Where(id => !string.IsNullOrEmpty(id)) + .Select(id => id!) + .ToList(); + deletedFiles.AddRange(identifiedDeleted); + + // A code-0 file/result entry with no id still counts as a confirmed delete. Pair leftover + // requested ids to those unidentified successes. An empty files/results list does not — + // AllSuccessful with no per-file rows must not be treated as deleted + var unidentifiedSuccessCount = successfulResults.Count - identifiedDeleted.Count; + if (unidentifiedSuccessCount > 0) + { + var remainingRequestedIds = chunk.Where(id => + !identifiedDeleted.Contains(id, StringComparer.OrdinalIgnoreCase) && + failedResults.All(f => !id.Equals(f.ResolvedFileId, StringComparison.OrdinalIgnoreCase))); + deletedFiles.AddRange(remainingRequestedIds.Take(unidentifiedSuccessCount)); + } } return new DeleteFilesResult diff --git a/backend/CPS.ComplexCases.Egress/Models/Response/DeleteFilesResponse.cs b/backend/CPS.ComplexCases.Egress/Models/Response/DeleteFilesResponse.cs index dfa7d158c..09239ab37 100644 --- a/backend/CPS.ComplexCases.Egress/Models/Response/DeleteFilesResponse.cs +++ b/backend/CPS.ComplexCases.Egress/Models/Response/DeleteFilesResponse.cs @@ -8,7 +8,11 @@ public class DeleteFilesResponse public bool AllSuccessful { get; set; } [JsonPropertyName("files")] public List Files { get; set; } = []; + [JsonPropertyName("results")] + public List Results { get; set; } = []; + [JsonIgnore] + public List Items => Results.Count > 0 ? Results : Files; } public class DeletedFileResult @@ -21,6 +25,16 @@ public class DeletedFileResult public string? Filename { get; set; } [JsonPropertyName("file_id")] public string? FileId { get; set; } + // Egress list/document APIs identify files as `id`. Bulk delete responses + // use the same field; `file_id` is kept for compatibility if it is sent. + [JsonPropertyName("id")] + public string? Id { get; set; } [JsonPropertyName("is_folder")] public bool IsFolder { get; set; } -} \ No newline at end of file + + [JsonIgnore] + public string? ResolvedFileId => + !string.IsNullOrWhiteSpace(FileId) ? FileId + : !string.IsNullOrWhiteSpace(Id) ? Id + : Filename; +} diff --git a/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/DeleteFilesTests.cs b/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/DeleteFilesTests.cs index b6d9847ce..6ce7aff88 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/DeleteFilesTests.cs +++ b/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/DeleteFilesTests.cs @@ -389,6 +389,32 @@ public async Task Run_RecordsAllFilesAsDeletionErrors_WhenDeletedFilesIsEmpty() Times.Once); } + [Fact] + public async Task Run_RecordsAllFilesAsDeletionErrors_WhenDeletedFilesIsEmptyAndNotAllSuccessful() + { + var payload = CreateEgressToNetAppPayload(); + var items = CreateCompletedItems(("file1.txt", "f1"), ("file2.txt", "f2")); + SetupDeleteRun(payload, items, new DeleteFilesResult + { + AllSuccessful = false, + DeletedFiles = [], + FailedFiles = [] + }); + + await _activity.Run(payload, _durableTaskClientStub, CancellationToken.None); + + _transferEntityHelperMock.Verify( + c => c.DeleteMovedItemsCompleted( + It.IsAny(), + payload.TransferId, + It.Is>(errors => + errors.Count == 2 && + errors.Any(e => e.FileId == "f1") && + errors.Any(e => e.FileId == "f2")), + It.IsAny()), + Times.Once); + } + [Fact] public async Task Run_RecordsMissingFilesAsDeletionErrors_WhenDeletedFilesCountIsLessThanRequested() { diff --git a/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/UpdateActivityLogTests.cs b/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/UpdateActivityLogTests.cs index 3019f5e53..1e59019a8 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/UpdateActivityLogTests.cs +++ b/backend/CPS.ComplexCases.FileTransfer.API.Tests/Unit/Durable/Activity/UpdateActivityLogTests.cs @@ -441,6 +441,69 @@ public async Task Run_IncludesDeletionErrors_WhenTransferTypeIsMoveAndDirectionI ), Times.Once); } + [Fact] + public async Task Run_MapsDeletionErrorFileIdToSourcePath_WhenSuccessfulItemExists() + { + var transferId = Guid.NewGuid(); + var userName = _fixture.Create(); + var caseId = _fixture.Create(); + const string fileId = "6a7b09840b11b5e3185286b7"; + const string sourcePath = "1. ABEs for Transcript/Free_Test_Data_10.5MB_PDF.pdf"; + + var entityState = new TransferEntity + { + Id = transferId, + CaseId = caseId, + Direction = TransferDirection.EgressToNetApp, + TransferType = TransferType.Move, + TotalFiles = 1, + BearerToken = _bearerToken, + SourcePaths = [new TransferSourcePath { FullFilePath = sourcePath, Path = sourcePath }], + SuccessfulItems = + [ + new TransferItem + { + SourcePath = sourcePath, + FileId = fileId, + Size = 11081517, + IsRenamed = false, + Status = TransferItemStatus.Completed + } + ], + DeletionErrors = + [ + new DeletionError { FileId = fileId, ErrorMessage = "File was not confirmed deleted by Egress." } + ], + DestinationPath = "/dest/path", + }; + + _durableEntityClientStub.OnGetEntityAsync = (_, _) => + Task.FromResult?>(new EntityMetadata( + new EntityInstanceId("TransferEntity", transferId.ToString()), + entityState + )); + + var payload = new UpdateActivityLogPayload + { + TransferId = transferId.ToString(), + ActionType = ActionType.TransferCompleted, + UserName = userName + }; + + await _activity.Run(payload, _durableTaskClientStub); + + _activityLogServiceMock.Verify(service => service.CreateActivityLogAsync( + payload.ActionType, + ResourceType.FileTransfer, + caseId, + entityState.Id.ToString(), + entityState.Direction.ToString(), + userName, + It.Is(doc => + doc.RootElement.GetProperty("errors")[0].GetProperty("path").GetString() == sourcePath) + ), Times.Once); + } + [Fact] public async Task Run_DoesNotIncludeDeletionErrors_WhenTransferTypeIsCopy() { diff --git a/backend/CPS.ComplexCases.FileTransfer.API/Attributes/HttpTelemetryExtension.cs b/backend/CPS.ComplexCases.FileTransfer.API/Attributes/HttpTelemetryExtension.cs new file mode 100644 index 000000000..36333686c --- /dev/null +++ b/backend/CPS.ComplexCases.FileTransfer.API/Attributes/HttpTelemetryExtension.cs @@ -0,0 +1,5 @@ +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + +// Load the host extension so HealthCheckTelemetryFilter can drop /api/status telemetry +// emitted by the Functions host (worker-side processors never see those items). +[assembly: ExtensionInformation("CPS.ComplexCases.API.HttpTelemetry", "1.0.42")] diff --git a/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj b/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj index d7ae3d700..b67299351 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj +++ b/backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj @@ -11,6 +11,7 @@ + @@ -43,4 +44,4 @@ Never - \ No newline at end of file + diff --git a/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/DeleteFiles.cs b/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/DeleteFiles.cs index b2710d0b3..bc91514b7 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/DeleteFiles.cs +++ b/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/DeleteFiles.cs @@ -79,10 +79,13 @@ public async Task Run([ActivityTrigger] DeleteFilesPayload? payload, [DurableCli if (deletionErrors.Count != 0) { _logger.LogWarning( - "Failed to delete {FailedCount} of {RequestedCount} files for transfer ID {TransferId}.", + "Failed to delete {FailedCount} of {RequestedCount} files for transfer ID {TransferId}. AllSuccessful={AllSuccessful}, DeletedIdentifiers={DeletedCount}, FailedIdentifiers={FailedApiCount}.", deletionErrors.Count, filesToDelete.Count, - payload.TransferId); + payload.TransferId, + result.AllSuccessful, + (result.DeletedFiles ?? []).Count, + (result.FailedFiles ?? []).Count); } else { @@ -153,7 +156,15 @@ private static List BuildDeletionErrors(List f } } - foreach (var file in filesToDelete.Where(file => !IsAccountedFor(file, accountedIdentifiers))) + var unaccountedFiles = filesToDelete.Where(file => !IsAccountedFor(file, accountedIdentifiers)).ToList(); + if (unaccountedFiles.Count == 0) + { + return deletionErrors; + } + + // compare confirmed deletes against the requested count. AllSuccessful with + // no DeletedFiles (for example an unknown file id) must still record DeletionErrors. + foreach (var file in unaccountedFiles) { deletionErrors.Add(new DeletionError { diff --git a/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/UpdateActivityLog.cs b/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/UpdateActivityLog.cs index c574a4947..277f9e5b7 100644 --- a/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/UpdateActivityLog.cs +++ b/backend/CPS.ComplexCases.FileTransfer.API/Durable/Activity/UpdateActivityLog.cs @@ -116,7 +116,7 @@ public async Task Run([ActivityTrigger] UpdateActivityLogPayload payload, [Durab { deletionErrors = entity.State.DeletionErrors.Select(x => new FileTransferError { - Path = x.FileId, + Path = ResolveDeletionErrorPath(entity.State.SuccessfulItems, x.FileId), ErrorMessage = x.ErrorMessage }).ToList(); errorItems.AddRange(deletionErrors); @@ -170,4 +170,13 @@ private static string PrependNetappRootFolder(string netappRootFolderPath, strin return $"{root}/{relative}"; } + + private static string ResolveDeletionErrorPath(IEnumerable successfulItems, string fileId) + { + var matchingItem = successfulItems.FirstOrDefault(item => + !string.IsNullOrEmpty(item.FileId) && + item.FileId.Equals(fileId, StringComparison.OrdinalIgnoreCase)); + + return matchingItem?.SourcePath ?? fileId; + } } diff --git a/backend/CPS.ComplexCases.sln b/backend/CPS.ComplexCases.sln index b724ccd30..6d2e8bdb9 100644 --- a/backend/CPS.ComplexCases.sln +++ b/backend/CPS.ComplexCases.sln @@ -49,6 +49,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cps.ComplexCases.Data.Tests EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CPS.ComplexCases.API.Integration.Tests", "CPS.ComplexCases.API.Integration.Tests\CPS.ComplexCases.API.Integration.Tests.csproj", "{CDF9B98D-75FB-4F39-9AAD-AE3F7281153B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CPS.ComplexCases.API.HttpTelemetry.Tests", "CPS.ComplexCases.API.HttpTelemetry.Tests\CPS.ComplexCases.API.HttpTelemetry.Tests.csproj", "{ABB60364-BC8B-42DA-8DA5-734C6380501B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -335,6 +337,18 @@ Global {CDF9B98D-75FB-4F39-9AAD-AE3F7281153B}.Release|x64.Build.0 = Release|Any CPU {CDF9B98D-75FB-4F39-9AAD-AE3F7281153B}.Release|x86.ActiveCfg = Release|Any CPU {CDF9B98D-75FB-4F39-9AAD-AE3F7281153B}.Release|x86.Build.0 = Release|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Debug|x64.ActiveCfg = Debug|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Debug|x64.Build.0 = Debug|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Debug|x86.ActiveCfg = Debug|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Debug|x86.Build.0 = Debug|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Release|Any CPU.Build.0 = Release|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Release|x64.ActiveCfg = Release|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Release|x64.Build.0 = Release|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Release|x86.ActiveCfg = Release|Any CPU + {ABB60364-BC8B-42DA-8DA5-734C6380501B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/devops-pipelines/api-postman-test.yml b/devops-pipelines/api-postman-test.yml index 9ba4f5b40..837c6fc3c 100644 --- a/devops-pipelines/api-postman-test.yml +++ b/devops-pipelines/api-postman-test.yml @@ -1,6 +1,6 @@ schedules: - - cron: "0 5 * * Tue-Sat" - displayName: "Weekday 5am UTC Api Test" + - cron: "0 3 * * Tue-Sat" + displayName: "Weekday 3am UTC Api Test" branches: include: - main diff --git a/devops-pipelines/backend/backend-build-and-deploy.yml b/devops-pipelines/backend/backend-build-and-deploy.yml index 0ba2ccf87..102b22b8c 100644 --- a/devops-pipelines/backend/backend-build-and-deploy.yml +++ b/devops-pipelines/backend/backend-build-and-deploy.yml @@ -25,9 +25,9 @@ variables: - name: buildConfiguration value: "Release" - name: dotNetVersion - value: '10.x' + value: "10.x" - name: efVersion - value: '10.0.*' + value: "10.0.*" - name: asposeLicenceFileName value: "Aspose.Total.NET.lic" @@ -43,7 +43,6 @@ stages: parameters: projectPath: "backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj" projectName: "HttpTelemetry" - buildConfiguration: $(buildConfiguration) dotNetVersion: $(dotNetVersion) useLocalNuGet: false publishOutput: false @@ -68,7 +67,7 @@ stages: jobs: - ${{ if or(eq(parameters.deployMainApi, true), eq(parameters.deployDB, true)) }}: - job: Build_Main_API_And_DB - displayName: "Build Main API & DB Migration Scripts" + displayName: "Build Main API / DB Migration Scripts" steps: - ${{ if eq(parameters.deployMainApi, true) }}: - task: DownloadSecureFile@1 @@ -90,11 +89,12 @@ stages: parameters: projectPath: "backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj" projectName: "lacc-main-api" - buildConfiguration: $(buildConfiguration) dotNetVersion: $(dotNetVersion) useLocalNuGet: true ${{ if eq(parameters.deployMainApi, true) }}: publishOutput: true + releaseVersionArg: "/p:Version=$(releaseVersion)" + releaseSourceVersionArg: "/p:SourceRevisionId=$(Build.SourceVersion)" ${{ else }}: publishOutput: false runTests: false @@ -130,27 +130,18 @@ stages: parameters: projectPath: "backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj" projectName: "lacc-filetransfer-api" - buildConfiguration: $(buildConfiguration) dotNetVersion: $(dotNetVersion) useLocalNuGet: true publishOutput: true runTests: false + releaseVersionArg: "/p:Version=$(releaseVersion)" + releaseSourceVersionArg: "/p:SourceRevisionId=$(Build.SourceVersion)" # Deployment stages: - - template: ../templates/backend-deploy-stages.yml - parameters: - environment: dev - requireValidation: false - deployMainApi: ${{ parameters.deployMainApi }} - deployFiletransfer: ${{ parameters.deployFiletransfer }} - deployDB: ${{ parameters.deployDB }} - - template: ../templates/backend-deploy-stages.yml parameters: environment: staging - requireValidation: true - notifyUsers: $(preprodNotifyUsers) - approvers: $(preprodApprovers) + dependsOn: Build_Backend deployToSlot: true slotName: "stg" runPostmanSmokeTest: true @@ -158,19 +149,27 @@ stages: deployFiletransfer: ${{ parameters.deployFiletransfer }} deployDB: ${{ parameters.deployDB }} - - stage: Run_E2E_Tests - displayName: "Run E2E tests" + - stage: Release_Gate + displayName: "Release Gate" jobs: - - template: ../templates/e2e-playwright-test-job.yml - parameters: - environment: staging + - job: Manual_Validation + pool: server + timeoutInMinutes: 6500 + steps: + - task: ManualValidation@1 + displayName: Request Validation + timeoutInMinutes: 6480 + inputs: + notifyUsers: $(prodNotifyUsers) + approvers: $(prodApprovers) + allowApproversToApproveTheirOwnRuns: true + instructions: "Approval is required to deploy to production." + onTimeout: "reject" - template: ../templates/backend-deploy-stages.yml parameters: environment: prod - requireValidation: true - notifyUsers: $(prodNotifyUsers) - approvers: $(prodApprovers) + dependsOn: Release_Gate agentPool: "LACC Prod Pool" azureSubscription: "Azure Pipeline: Large and Complex Cases - Prod" deployToSlot: true diff --git a/devops-pipelines/backend/backend-dev-build-and-deploy.yml b/devops-pipelines/backend/backend-dev-build-and-deploy.yml new file mode 100644 index 000000000..851f1972e --- /dev/null +++ b/devops-pipelines/backend/backend-dev-build-and-deploy.yml @@ -0,0 +1,142 @@ +trigger: + batch: true + branches: + include: + - development + paths: + include: + - backend/* + +parameters: + - name: deployDB + type: boolean + default: true + - name: deployMainApi + type: boolean + default: true + - name: deployFiletransfer + type: boolean + default: true + +pr: none + +variables: + - name: buildConfiguration + value: "Release" + - name: dotNetVersion + value: "10.x" + - name: efVersion + value: "10.0.*" + - name: asposeLicenceFileName + value: "Aspose.Total.NET.lic" + +stages: + - stage: Build_Local_NuGet + displayName: "Pack and Publish NuGet Package" + pool: "LACC PreProd Pool" + jobs: + - job: Package_NuGet + displayName: "Pack and Publish NuGet Package" + steps: + - template: ../templates/dotnet-build-steps.yml + parameters: + projectPath: "backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj" + projectName: "HttpTelemetry" + dotNetVersion: $(dotNetVersion) + useLocalNuGet: false + publishOutput: false + runTests: false + + - task: DotNetCoreCLI@2 + displayName: "Pack HttpTelemetry" + inputs: + command: "pack" + packagesToPack: "backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj" + outputDir: "$(Build.ArtifactStagingDirectory)" + arguments: "--configuration $(buildConfiguration) --no-build" + + - publish: "$(Build.ArtifactStagingDirectory)" + artifact: "localnuget" + displayName: "Publish HttpTelemetry Package" + + - stage: Build_Backend + displayName: "Generate Build Artifacts for Deployment" + dependsOn: Build_Local_NuGet + pool: "LACC PreProd Pool" + jobs: + - ${{ if or(eq(parameters.deployMainApi, true), eq(parameters.deployDB, true)) }}: + - job: Build_Main_API_And_DB + displayName: "Build Main API / DB Migration Scripts" + steps: + - ${{ if eq(parameters.deployMainApi, true) }}: + - task: DownloadSecureFile@1 + inputs: + secureFile: $(asposeLicenceFileName) + displayName: "Download Aspose Licence File" + + - task: CopyFiles@2 + inputs: + sourceFolder: $(Agent.TempDirectory) + contents: $(asposeLicenceFileName) + targetFolder: "backend/CPS.ComplexCases.API/Licences" + OverWrite: true + displayName: "Copy Aspose Licence File to Project Folder" + + # When buildMainApi=false, the project still needs building + # as the startup project for the database migration script + - template: ../templates/dotnet-build-steps.yml + parameters: + projectPath: "backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj" + projectName: "lacc-main-api" + dotNetVersion: $(dotNetVersion) + useLocalNuGet: true + ${{ if eq(parameters.deployMainApi, true) }}: + publishOutput: true + ${{ else }}: + publishOutput: false + runTests: false + + - ${{ if eq(parameters.deployDB, true) }}: + - task: DotNetCoreCLI@2 + displayName: "Build Database Project" + inputs: + command: "build" + projects: "backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj" + arguments: "--configuration $(buildConfiguration)" + + - task: Bash@3 + displayName: "Generate Migration Scripts" + inputs: + filePath: "$(System.DefaultWorkingDirectory)/devops-pipelines/scripts/generateDBMigrationScripts.sh" + env: + EF_VERSION: $(efVersion) + DATA_PROJECT_PATH: "backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj" + STARTUP_PROJECT_PATH: "backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj" + MIGRATION_SCRIPT_OUT: "database-artifact/migration-script.sql" + ROLLBACK_SCRIPT_OUT: "database-artifact/rollback-migration-script.sql" + + - publish: "database-artifact" + artifact: "database-scripts-drop" + displayName: "Publish Database Scripts" + + - ${{ if eq(parameters.deployFiletransfer, true) }}: + - job: Build_FileTransfer_API + displayName: "Build FileTransfer API" + steps: + - template: ../templates/dotnet-build-steps.yml + parameters: + projectPath: "backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj" + projectName: "lacc-filetransfer-api" + dotNetVersion: $(dotNetVersion) + useLocalNuGet: true + publishOutput: true + runTests: false + + # Deploy to the dev environment + - template: ../templates/backend-deploy-stages.yml + parameters: + environment: dev + dependsOn: Build_Backend + deployMainApi: ${{ parameters.deployMainApi }} + deployFiletransfer: ${{ parameters.deployFiletransfer }} + deployDB: ${{ parameters.deployDB }} diff --git a/devops-pipelines/backend/backend-pr-build-and-test.yml b/devops-pipelines/backend/backend-pr-build-and-test.yml index 060ce4342..7a31b6ad5 100644 --- a/devops-pipelines/backend/backend-pr-build-and-test.yml +++ b/devops-pipelines/backend/backend-pr-build-and-test.yml @@ -4,21 +4,11 @@ pr: branches: include: - main + - development paths: include: - backend/* -parameters: - - name: buildDB - type: boolean - default: true - - name: buildMainApi - type: boolean - default: true - - name: buildFiletransfer - type: boolean - default: true - variables: - name: buildConfiguration value: "Release" @@ -42,7 +32,6 @@ stages: parameters: projectPath: "backend/CPS.ComplexCases.API.HttpTelemetry/CPS.ComplexCases.API.HttpTelemetry.csproj" projectName: "HttpTelemetry" - buildConfiguration: $(buildConfiguration) dotNetVersion: $(dotNetVersion) useLocalNuGet: false publishOutput: false @@ -68,70 +57,9 @@ stages: parameters: projectPath: "backend/CPS.ComplexCases.sln" projectName: "Backend Solution" - buildConfiguration: "$(buildConfiguration)" dotNetVersion: $(dotNetVersion) useLocalNuGet: true publishOutput: false runTests: true checkFormatting: true testResultsDirectory: $(testResultsDirectory) - - - ${{ if or(eq(parameters.buildMainApi, true), eq(parameters.buildDB, true)) }}: - - job: Build_Main_API_And_DB - displayName: "Build Main API & Data Project" - dependsOn: Build_And_Test_Solution - steps: - # Build and Main API - also required for generating database migration script - - template: ../templates/dotnet-build-steps.yml - parameters: - projectPath: "backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj" - projectName: "lacc-main-api-drop" - buildConfiguration: $(buildConfiguration) - dotNetVersion: $(dotNetVersion) - useLocalNuGet: true - publishOutput: false - runTests: false - - - ${{ if eq(parameters.buildDB, true) }}: - # Build Data project with dependencies - - task: DotNetCoreCLI@2 - displayName: "Build Database Project" - inputs: - command: "build" - projects: "backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj" - arguments: "--configuration $(buildConfiguration)" - - - task: Bash@3 - displayName: "Generate Migration Scripts" - inputs: - targetType: "inline" - script: | - export PATH="$PATH:$HOME/.dotnet/tools" - - if ! dotnet ef --version &> /dev/null; then - dotnet tool install --global dotnet-ef --version $(efVersion) - fi - - dotnet ef migrations script \ - --idempotent \ - --no-build \ - --configuration $(buildConfiguration) \ - --output $(Build.ArtifactStagingDirectory)/migration-script.sql \ - --project backend/CPS.ComplexCases.Data/CPS.ComplexCases.Data.csproj \ - --startup-project backend/CPS.ComplexCases.API/CPS.ComplexCases.API.csproj - - - ${{ if eq(parameters.buildFiletransfer, true) }}: - - job: Build_FileTransfer_API - displayName: "Build FileTransfer API" - dependsOn: Build_And_Test_Solution - steps: - # Build and publish FileTransfer API - - template: ../templates/dotnet-build-steps.yml - parameters: - projectPath: "backend/CPS.ComplexCases.FileTransfer.API/CPS.ComplexCases.FileTransfer.API.csproj" - projectName: "lacc-filetransfer-api-drop" - buildConfiguration: $(buildConfiguration) - dotNetVersion: $(dotNetVersion) - useLocalNuGet: false - publishOutput: false - runTests: false diff --git a/devops-pipelines/backend/backend-pr-integration-tests.yml b/devops-pipelines/backend/backend-pr-integration-tests.yml index e23bd5e10..6efbf8b40 100644 --- a/devops-pipelines/backend/backend-pr-integration-tests.yml +++ b/devops-pipelines/backend/backend-pr-integration-tests.yml @@ -3,7 +3,7 @@ trigger: none pr: branches: include: - - main + - development paths: include: - backend/* diff --git a/devops-pipelines/backend/db-rollback.yml b/devops-pipelines/backend/db-rollback.yml index 90f35fd72..f234eed93 100644 --- a/devops-pipelines/backend/db-rollback.yml +++ b/devops-pipelines/backend/db-rollback.yml @@ -7,20 +7,17 @@ parameters: - name: environment displayName: 'Target Environment' type: string - default: 'dev' values: - dev - staging - prod - name: agentPool type: string - default: 'LACC PreProd Pool' values: - 'LACC PreProd Pool' - 'LACC Prod Pool' - name: azureSubscription type: string - default: 'Azure Pipeline: Large and Complex Cases - Pre-Prod' values: - 'Azure Pipeline: Large and Complex Cases - Pre-Prod' - 'Azure Pipeline: Large and Complex Cases - Prod' diff --git a/devops-pipelines/cleanup-testfiles.yml b/devops-pipelines/cleanup-testfiles.yml index 6e25ba228..0aa4cc955 100644 --- a/devops-pipelines/cleanup-testfiles.yml +++ b/devops-pipelines/cleanup-testfiles.yml @@ -1,6 +1,6 @@ schedules: - - cron: "0 3 * * Tue-Sat" - displayName: "Weekday 3am UTC cleanup" + - cron: "0 5 * * Tue-Sat" + displayName: "Weekday 5am UTC cleanup" branches: include: - main diff --git a/devops-pipelines/templates/backend-deploy-stages.yml b/devops-pipelines/templates/backend-deploy-stages.yml index 986072de3..ee5514401 100644 --- a/devops-pipelines/templates/backend-deploy-stages.yml +++ b/devops-pipelines/templates/backend-deploy-stages.yml @@ -13,15 +13,6 @@ parameters: values: - "LACC PreProd Pool" - "LACC Prod Pool" - - name: requireValidation - type: boolean - default: false - - name: notifyUsers - type: string - default: "" - - name: approvers - type: string - default: "" - name: azureSubscription type: string default: "Azure Pipeline: Large and Complex Cases - Pre-Prod" @@ -34,9 +25,9 @@ parameters: - name: runPostmanSmokeTest type: boolean default: false - - name: buildStageName + - name: dependsOn type: string - default: "Build_Backend" + default: "" - name: deployDB type: boolean - name: deployMainApi @@ -45,26 +36,9 @@ parameters: type: boolean stages: - - ${{ if eq(parameters.requireValidation, true) }}: - - stage: Approve_Deployment_${{ parameters.environment }} - displayName: "${{ parameters.environment }} - Approve Deployment" - jobs: - - template: ./manual-validation-jobs.yml - parameters: - instructions: "Please verify the deployment to the lower environment worked as expected before approving deployment to ${{parameters.environment}}." - notifyUsers: ${{ parameters.notifyUsers }} - approvers: ${{ parameters.approvers }} - agentPool: ${{ parameters.agentPool }} - - stage: Deploy_Backend_${{ parameters.environment }} - ${{ if eq(parameters.requireValidation, true) }}: - dependsOn: - - Approve_Deployment_${{ parameters.environment }} - - ${{ parameters.buildStageName }} - ${{ else }}: - dependsOn: - - ${{ parameters.buildStageName }} displayName: "${{ parameters.environment }} - Deploy Backend Services" + dependsOn: ${{ parameters.dependsOn }} variables: - template: ./variables/backend-deploy-vars.yml parameters: diff --git a/devops-pipelines/templates/dotnet-build-steps.yml b/devops-pipelines/templates/dotnet-build-steps.yml index 2ddd92203..306b3507e 100644 --- a/devops-pipelines/templates/dotnet-build-steps.yml +++ b/devops-pipelines/templates/dotnet-build-steps.yml @@ -1,15 +1,46 @@ parameters: - projectPath: "" - projectName: "" - buildConfiguration: "Release" - dotNetVersion: "10.x" - useLocalNuGet: false - nugetArtifact: "localnuget" - nugetArtifactDownloadPath: "$(Build.SourcesDirectory)/localnuget" - publishOutput: false - runTests: false - checkFormatting: false - testResultsDirectory: "" + - name: projectPath + type: string + default: "" + - name: projectName + type: string + default: "" + - name: dotNetVersion + type: string + default: "10.x" + - name: useLocalNuGet + type: boolean + default: false + - name: nugetArtifact + type: string + default: "localnuget" + - name: nugetArtifactDownloadPath + type: string + default: "$(Build.SourcesDirectory)/localnuget" + - name: publishOutput + type: boolean + default: false + - name: runTests + type: boolean + default: false + - name: checkFormatting + type: boolean + default: false + - name: testResultsDirectory + type: string + default: "" + - name: releaseVersionArg + type: string + default: " " + values: + - " " + - "/p:Version=$(releaseVersion)" + - name: releaseSourceVersionArg + type: string + default: " " + values: + - " " + - "/p:SourceRevisionId=$(Build.SourceVersion)" steps: - task: UseDotNet@2 @@ -98,7 +129,11 @@ steps: inputs: command: "build" projects: "${{ parameters.projectPath }}" - arguments: "--configuration ${{ parameters.buildConfiguration }} --no-restore" + arguments: > + --configuration $(buildConfiguration) + --no-restore + ${{ parameters.releaseSourceVersionArg }} + ${{ parameters.releaseVersionArg }} # Run tests if enabled - ${{ if eq(parameters.runTests, true) }}: @@ -108,7 +143,7 @@ steps: command: "test" projects: "${{ parameters.projectPath }}" arguments: > - --configuration ${{ parameters.buildConfiguration }} + --configuration $(buildConfiguration) --no-build --collect "XPlat Code Coverage;Format=cobertura" --settings "$(Build.SourcesDirectory)/backend/CodeCoverage.runsettings" @@ -126,7 +161,7 @@ steps: failIfCoverageEmpty: true condition: succeededOrFailed() - # Publish if enabled + # Publish if enabled, no build or restore as these are already done in previous steps - ${{ if eq(parameters.publishOutput, true) }}: - task: DotNetCoreCLI@2 displayName: "Dotnet Publish ${{ parameters.projectName }}" @@ -135,12 +170,12 @@ steps: projects: "${{ parameters.projectPath }}" publishWebProjects: false arguments: > - --configuration ${{ parameters.buildConfiguration }} + --configuration $(buildConfiguration) --output $(Build.ArtifactStagingDirectory)/${{ parameters.projectName }} --no-restore + --no-build --self-contained false /p:PublishReadyToRun=false - /p:SourceRevisionId=$(Build.SourceVersion) - publish: "$(Build.ArtifactStagingDirectory)/${{ parameters.projectName }}" artifact: "${{ parameters.projectName }}-drop" diff --git a/devops-pipelines/templates/e2e-playwright-test-job.yml b/devops-pipelines/templates/e2e-playwright-test-job.yml index b61cc5d01..d8c4fc0ab 100644 --- a/devops-pipelines/templates/e2e-playwright-test-job.yml +++ b/devops-pipelines/templates/e2e-playwright-test-job.yml @@ -27,8 +27,6 @@ jobs: value: "22.x" - name: workingDir value: "$(System.DefaultWorkingDirectory)/e2e/pw" - - name: transferMaterialsV1 # Staging renders the v1 ("Shared Drive") transfer screen for the E2E user. - value: ${{ ne(parameters.environment, 'dev') }} steps: - task: UseNode@1 @@ -51,7 +49,7 @@ jobs: displayName: "Run E2E Tests" env: CI: true - TRANSFER_MATERIALS_V1: $(transferMaterialsV1) + TRANSFER_MATERIALS_V1: true BASE_URL: $(RedirectUrlLccUi) CMS_LOGIN_PAGE: "$(LccApiBaseUrl)/api/tactical/login" EGRESS_BASE_URL: $(EgressOptionsUrl) diff --git a/devops-pipelines/templates/ui-build-and-deploy-stages.yml b/devops-pipelines/templates/ui-build-and-deploy-stages.yml index b86603805..9ec245367 100644 --- a/devops-pipelines/templates/ui-build-and-deploy-stages.yml +++ b/devops-pipelines/templates/ui-build-and-deploy-stages.yml @@ -13,15 +13,6 @@ parameters: - name: azureSubscription type: string default: "Azure Pipeline: Large and Complex Cases - Pre-Prod" - - name: requireValidation - type: boolean - default: false - - name: notifyUsers - type: string - default: "" - - name: approvers - type: string - default: "" - name: deployToSlot type: boolean default: false @@ -30,21 +21,8 @@ parameters: default: "" stages: - - ${{ if eq(parameters.requireValidation, true) }}: - - stage: Approve_Deployment_${{ parameters.environment }} - displayName: "LACC ${{ parameters.environment }} - Approve Deployment" - jobs: - - template: ./manual-validation-jobs.yml - parameters: - instructions: "Please verify the deployment to the lower environment worked as expected before approving deployment to ${{parameters.environment}}." - notifyUsers: ${{ parameters.notifyUsers }} - approvers: ${{ parameters.approvers }} - agentPool: ${{ parameters.agentPool }} - - stage: Build_Deploy_${{ parameters.environment }} displayName: "${{ parameters.environment }} - Build & Deploy Web App" - ${{ if eq(parameters.requireValidation, true) }}: - dependsOn: Approve_Deployment_${{ parameters.environment }} variables: buildArtifact: "lacc-ui-${{ parameters.environment }}-drop" jobs: diff --git a/devops-pipelines/ui/ui-build-and-deploy.yml b/devops-pipelines/ui/ui-build-and-deploy.yml index 915d4a844..4472fc707 100644 --- a/devops-pipelines/ui/ui-build-and-deploy.yml +++ b/devops-pipelines/ui/ui-build-and-deploy.yml @@ -12,33 +12,32 @@ variables: - group: lacc-deployment-global stages: - - template: ../templates/ui-build-and-deploy-stages.yml - parameters: - environment: dev - requireValidation: false - - template: ../templates/ui-build-and-deploy-stages.yml parameters: environment: staging - requireValidation: true - notifyUsers: $(preprodNotifyUsers) - approvers: $(preprodApprovers) deployToSlot: true slotName: "stg" - - stage: Run_E2E_Tests - displayName: "Run E2E tests" + - stage: Release_Gate + displayName: "Release Gate" jobs: - - template: ../templates/e2e-playwright-test-job.yml - parameters: - environment: staging + - job: Manual_Validation + pool: server + timeoutInMinutes: 6500 + steps: + - task: ManualValidation@1 + displayName: Request Validation + timeoutInMinutes: 6480 + inputs: + notifyUsers: $(prodNotifyUsers) + approvers: $(prodApprovers) + allowApproversToApproveTheirOwnRuns: true + instructions: "Approval is required to deploy to production." + onTimeout: "reject" - template: ../templates/ui-build-and-deploy-stages.yml parameters: environment: prod - requireValidation: true - notifyUsers: $(prodNotifyUsers) - approvers: $(prodApprovers) agentPool: "LACC Prod Pool" azureSubscription: "Azure Pipeline: Large and Complex Cases - Prod" deployToSlot: true diff --git a/devops-pipelines/ui/ui-dev-build-and-deploy.yml b/devops-pipelines/ui/ui-dev-build-and-deploy.yml new file mode 100644 index 000000000..b81645374 --- /dev/null +++ b/devops-pipelines/ui/ui-dev-build-and-deploy.yml @@ -0,0 +1,14 @@ +trigger: + branches: + include: + - development + paths: + include: + - ui-spa/* + +pr: none + +stages: + - template: ../templates/ui-build-and-deploy-stages.yml + parameters: + environment: dev diff --git a/devops-pipelines/ui/ui-pr-build-and-test.yml b/devops-pipelines/ui/ui-pr-build-and-test.yml index fb1f81ab9..fca90c3cc 100644 --- a/devops-pipelines/ui/ui-pr-build-and-test.yml +++ b/devops-pipelines/ui/ui-pr-build-and-test.yml @@ -4,6 +4,7 @@ pr: branches: include: - main + - development paths: include: - ui-spa/* diff --git a/ui-spa/playwright/integration-tests/case-search.spec.ts b/ui-spa/playwright/integration-tests/case-search.spec.ts index f592b0f60..ab6949c68 100644 --- a/ui-spa/playwright/integration-tests/case-search.spec.ts +++ b/ui-spa/playwright/integration-tests/case-search.spec.ts @@ -316,11 +316,12 @@ test.describe("Case Search/Results", async () => { ).not.toBeVisible(); await expect(page.getByText("Loading...")).not.toBeVisible(); await expect(page).toHaveURL("/unauthorised"); - await expect(page.locator("h1")).toHaveText("Unauthorised"); - await expect(page.getByText("Your session has timed out.")).toBeVisible(); + await expect(page.locator("h1")).toHaveText( + "You cannot access this service", + ); await expect( page.getByText( - "Please close this tab and re-launch the homepage screen from CMS Classic.", + "Close this tab and open the homepage again from CMS Classic.", ), ).toBeVisible(); }); @@ -669,11 +670,12 @@ test.describe("Case Search/Results", async () => { await expect(page.getByText("Loading...")).toBeVisible(); await expect(page.getByText("Loading...")).not.toBeVisible(); await expect(page).toHaveURL("/unauthorised"); - await expect(page.locator("h1")).toHaveText("Unauthorised"); - await expect(page.getByText("Your session has timed out.")).toBeVisible(); + await expect(page.locator("h1")).toHaveText( + "You cannot access this service", + ); await expect( page.getByText( - "Please close this tab and re-launch the homepage screen from CMS Classic.", + "Close this tab and open the homepage again from CMS Classic.", ), ).toBeVisible(); }); diff --git a/ui-spa/src/components/unauthorised/index.tsx b/ui-spa/src/components/unauthorised/index.tsx index cdef74156..453f1385f 100644 --- a/ui-spa/src/components/unauthorised/index.tsx +++ b/ui-spa/src/components/unauthorised/index.tsx @@ -1,12 +1,8 @@ const UnAuthorisedPage = () => { return (
-

Unauthorised

-

Your session has timed out.

-

- Please close this tab and re-launch the homepage screen from CMS - Classic. -

+

You cannot access this service

+

Close this tab and open the homepage again from CMS Classic.

); }; From f86801eb25efa07b372c412a538b2337994a5b52 Mon Sep 17 00:00:00 2001 From: Hasan Ahmed Date: Wed, 23 Sep 2026 15:25:45 +0100 Subject: [PATCH 2/6] Back merge main into development (#592) * filtering status telemetry (#581) * filtering status telemetry * fix nuget * pr comment fixes * story/FCT2-20672 : Text content update for the unauthorised page (#584) * task/FCT2-21994 backend test pipeline - rm redundant build job (#585) * egress delete check all files successful (#586) * Fct2 21924 bulk delete unknown files (#587) * deletion of unexisting files * deletion of existing files --------- Co-authored-by: Rhys Bridges Co-authored-by: Renjith Abby Co-authored-by: Lilach From 97756fb0896984565f24ef5f36b2563af69d0cc8 Mon Sep 17 00:00:00 2001 From: Lilach Date: Fri, 25 Sep 2026 14:30:47 +0100 Subject: [PATCH 3/6] subtask/FCT2-22142 Add separate test flow for PRs from development into main (#594) - Introduce prSourceBranch and prTargetBranch variables to conditionally run pipeline stages. - When a PR is opened from development into main: - Unit and integration tests are skipped. - If there are any changed files within the ui-spa directory, the playwright E2E tests will run against the DEV environment - When a PR is opened against development OR against main from any source other than development, only the relevant unit and integration tests will run. --- .../backend/backend-pr-build-and-test.yml | 8 +++++++ devops-pipelines/ui/ui-pr-build-and-test.yml | 21 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/devops-pipelines/backend/backend-pr-build-and-test.yml b/devops-pipelines/backend/backend-pr-build-and-test.yml index 7a31b6ad5..ac382fefb 100644 --- a/devops-pipelines/backend/backend-pr-build-and-test.yml +++ b/devops-pipelines/backend/backend-pr-build-and-test.yml @@ -18,11 +18,19 @@ variables: value: '10.0.*' - name: testResultsDirectory value: "$(Agent.TempDirectory)/TestResults" + - name: prSourceBranch + value: $[variables['System.PullRequest.SourceBranch']] + - name: prTargetBranch + value: $[variables['System.PullRequest.TargetBranch']] pool: "LACC PreProd Pool" stages: - stage: Build_And_Test_Backend + condition: not(and( + eq(variables.prSourceBranch, 'development' ), + eq(variables.prTargetBranch, 'main') + )) displayName: "Build, Test & Package" jobs: - job: Package_NuGet diff --git a/devops-pipelines/ui/ui-pr-build-and-test.yml b/devops-pipelines/ui/ui-pr-build-and-test.yml index fca90c3cc..029e3e7d3 100644 --- a/devops-pipelines/ui/ui-pr-build-and-test.yml +++ b/devops-pipelines/ui/ui-pr-build-and-test.yml @@ -18,9 +18,17 @@ variables: value: ui-spa - name: covDir value: "coverage" + - name: prSourceBranch + value: $[variables['System.PullRequest.SourceBranch']] + - name: prTargetBranch + value: $[variables['System.PullRequest.TargetBranch']] stages: - stage: Build_and_Test_UI + condition: not(and( + eq(variables.prSourceBranch, 'development' ), + eq(variables.prTargetBranch, 'main') + )) displayName: "Build & Test UI SPA" jobs: - job: Build_and_Test @@ -143,3 +151,16 @@ stages: inputs: summaryFileLocation: "$(workingDir)/$(covDir)/merged/Cobertura.xml" failIfCoverageEmpty: true + + # On PR development -> main, run the E2E tests in dev + - stage: Run_E2E_tests + condition: and( + eq(variables.prSourceBranch, 'development' ), + eq(variables.prTargetBranch, 'main' ) + ) + dependsOn: [] + displayName: "Run E2E Test on Dev" + jobs: + - template: ../templates/e2e-playwright-test-job.yml + parameters: + environment: "dev" From 6c50144ede1fe64da11ca7119c453094ff55cfd1 Mon Sep 17 00:00:00 2001 From: Lilach Date: Tue, 29 Sep 2026 15:03:58 +0100 Subject: [PATCH 4/6] Validate releaseVersion avar on PR and tag repo on merge to main --- devops-pipelines/tag-repo.yaml | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 devops-pipelines/tag-repo.yaml diff --git a/devops-pipelines/tag-repo.yaml b/devops-pipelines/tag-repo.yaml new file mode 100644 index 000000000..81c4292dd --- /dev/null +++ b/devops-pipelines/tag-repo.yaml @@ -0,0 +1,64 @@ +trigger: + branches: + include: + - main + +pr: + branches: + include: + - main + +variables: + - group: lacc-deployment-global + - name: buildReason + value: $[variables['Build.Reason']] + +pool: "LaCC PreProd Pool" + +steps: + - checkout: self + persistCredentials: true + + - script: | + set -euo pipefail + + if [[ ! "$(releaseVersion)" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "##vso[task.logissue type=error]The value of variable releaseVersion: $(releaseVersion) is not a valid semantic version. Expected format: MAJOR.MINOR.PATCH." + exit 1 + fi + + newTag="v$(releaseVersion)" + + # get all tags from origin and sort by version (ignore annotated tags peeled references) + latestTag=$(git ls-remote --tags origin 'refs/tags/v*' | + awk '$2 !~ /\^\{\}$/ {sub("refs/tags/", "", $2); print $2}' | + sort -V | + tail -n 1) + + if [[ ! "$latestTag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "##vso[task.logissue type=error]The latest tag found in the remote repository has an unexpected format: $latestTag. Expected format: vMAJOR.MINOR.PATCH." + exit 1 + fi + + echo "Latest tag: $latestTag" + echo "New tag: $newTag" + + if [[ "$(printf '%s\n%s\n' "$latestTag" "$newTag" | sort -V | tail -n1)" == "$latestTag" ]]; then + echo "##vso[task.logissue type=error]The semver value of variable releaseVersion: $(releaseVersion) must be greater than ${latestTag#v}." + exit 1 + else + echo "The respository will be tagged with the new release version $newTag" + echo "##vso[task.setvariable variable=newTag;]$newTag" + fi + displayName: "Validate releaseVersion Tag" + + - script: | + set -euo pipefail + + # overwrite any local tags. We are only interested in the remote: + git tag -f $(newTag) + + git push origin refs/tags/$(newTag) + # only tag repo on merge to main: + condition: and(succeeded(), eq(variables.buildReason, 'IndividualCI')) + displayName: "Tag Repository" From b336821def246c6f602c06646be4d0dff881f751 Mon Sep 17 00:00:00 2001 From: Lilach Date: Tue, 29 Sep 2026 15:40:57 +0100 Subject: [PATCH 5/6] handle case where no remote tags exist --- devops-pipelines/tag-repo.yaml | 35 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/devops-pipelines/tag-repo.yaml b/devops-pipelines/tag-repo.yaml index 81c4292dd..cc325a594 100644 --- a/devops-pipelines/tag-repo.yaml +++ b/devops-pipelines/tag-repo.yaml @@ -29,27 +29,36 @@ steps: newTag="v$(releaseVersion)" - # get all tags from origin and sort by version (ignore annotated tags peeled references) + # get the highest release tag from origin + # ignore peeled references produced for annotated tags latestTag=$(git ls-remote --tags origin 'refs/tags/v*' | awk '$2 !~ /\^\{\}$/ {sub("refs/tags/", "", $2); print $2}' | sort -V | tail -n 1) - if [[ ! "$latestTag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "##vso[task.logissue type=error]The latest tag found in the remote repository has an unexpected format: $latestTag. Expected format: vMAJOR.MINOR.PATCH." - exit 1 - fi + if [[ -z "$latestTag" ]]; then + echo "##vso[task.logissue type=warning]No release tags found in the remote repository." + else + echo "Latest remote tag: $latestTag" + echo "New release tag: $newTag" - echo "Latest tag: $latestTag" - echo "New tag: $newTag" + # validate the latest remote tag format + if [[ ! "$latestTag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "##vso[task.logissue type=error]The latest tag found in the remote repository has an unexpected format: $latestTag. Expected format: vMAJOR.MINOR.PATCH." + exit 1 + fi - if [[ "$(printf '%s\n%s\n' "$latestTag" "$newTag" | sort -V | tail -n1)" == "$latestTag" ]]; then - echo "##vso[task.logissue type=error]The semver value of variable releaseVersion: $(releaseVersion) must be greater than ${latestTag#v}." - exit 1 - else - echo "The respository will be tagged with the new release version $newTag" - echo "##vso[task.setvariable variable=newTag;]$newTag" + # fail if newTag is less than or equal to latestTag. + highestVersion=$(printf '%s\n%s\n' "$latestTag" "$newTag" | sort -V | tail -n 1) + + if [[ "$highestVersion" == "$latestTag" ]]; then + echo "##vso[task.logissue type=error]The semver value of variable releaseVersion: $(releaseVersion) must be greater than ${latestTag#v}." + exit 1 + fi fi + + echo "The respository will be tagged with the new release version $newTag" + echo "##vso[task.setvariable variable=newTag;]$newTag" displayName: "Validate releaseVersion Tag" - script: | From c80ce2509be3e5b22e6963274666556fc3730168 Mon Sep 17 00:00:00 2001 From: Lilach Date: Tue, 29 Sep 2026 15:08:36 +0000 Subject: [PATCH 6/6] only trigger pipeline on specified paths --- devops-pipelines/tag-repo.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/devops-pipelines/tag-repo.yaml b/devops-pipelines/tag-repo.yaml index cc325a594..c72509fc9 100644 --- a/devops-pipelines/tag-repo.yaml +++ b/devops-pipelines/tag-repo.yaml @@ -2,11 +2,19 @@ trigger: branches: include: - main + paths: + include: + - backend + - ui-spa pr: branches: include: - main + paths: + include: + - backend + - ui-spa variables: - group: lacc-deployment-global @@ -68,6 +76,6 @@ steps: git tag -f $(newTag) git push origin refs/tags/$(newTag) - # only tag repo on merge to main: + # only tag repo on merge to main condition: and(succeeded(), eq(variables.buildReason, 'IndividualCI')) displayName: "Tag Repository"