diff --git a/eng/pipelines/scripts/Stage-AutoReleaseSignedArtifacts.ps1 b/eng/pipelines/scripts/Stage-AutoReleaseSignedArtifacts.ps1 new file mode 100644 index 000000000000..786182284176 --- /dev/null +++ b/eng/pipelines/scripts/Stage-AutoReleaseSignedArtifacts.ps1 @@ -0,0 +1,76 @@ +#Requires -Version 7.0 +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $SourceDirectory, + [Parameter(Mandatory = $true)] + [string] $TargetDirectory, + [Parameter(Mandatory = $true)] + [string] $ArtifactsJson +) + +$ErrorActionPreference = "Stop" + +# MavenPackaging.ps1 (Get-MavenPackageDetails, etc.) lives in eng/scripts/, while this script is in +# eng/pipelines/scripts/ -- go up two levels to the eng/ root, then into scripts/. +. (Join-Path $PSScriptRoot ".." ".." "scripts" "MavenPackaging.ps1") + +$SourceDirectory = Resolve-Path $SourceDirectory +$TargetDirectory = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($TargetDirectory) + +$artifacts = @($ArtifactsJson | ConvertFrom-Json) + +if ($artifacts.Count -eq 0) { + throw "Auto-release artifact list was empty." +} + +if (Test-Path $TargetDirectory) { + Remove-Item -Path $TargetDirectory -Force -Recurse +} + +New-Item -ItemType Directory -Path $TargetDirectory -Force | Out-Null + +$packageDetails = Get-MavenPackageDetails -ArtifactDirectory $SourceDirectory +$artifactKeys = @($artifacts | ForEach-Object { "$($_.groupId)/$($_.name)" }) +$packagesToStage = @($packageDetails | Where-Object { "$($_.GroupID)/$($_.ArtifactID)" -in $artifactKeys }) + +Write-Host "Found $($packageDetails.Count) packages in '$SourceDirectory'." +Write-Host "$($packagesToStage.Count) packages match the auto-release artifact list." + +if ($packagesToStage.Count -ne $artifacts.Count) { + $foundKeys = @($packagesToStage | ForEach-Object { "$($_.GroupID)/$($_.ArtifactID)" }) + $missingKeys = @($artifactKeys | Where-Object { $_ -notin $foundKeys }) + throw "Unable to find signed artifacts for: $($missingKeys -join ', ')" +} + +foreach ($packageDetail in $packagesToStage) { + $groupIdDirectory = New-Item -Type Directory -Path $TargetDirectory -Name $packageDetail.GroupID -Force + $artifactIdDirectory = New-Item -Type Directory -Path $groupIdDirectory -Name $packageDetail.ArtifactID -Force + + Write-Host "Copying package $($packageDetail.FullyQualifiedName) to '$artifactIdDirectory'." + foreach ($associatedArtifact in $packageDetail.AssociatedArtifacts) { + Copy-Item -Path $associatedArtifact.File -Destination $artifactIdDirectory + } +} + +$sourcePackageInfoDirectory = Join-Path $SourceDirectory "PackageInfo" +$targetPackageInfoDirectory = New-Item -Type Directory -Path $TargetDirectory -Name "PackageInfo" -Force + +$missingPackageInfo = @() +foreach ($artifact in $artifacts) { + $packageInfoFile = Join-Path $sourcePackageInfoDirectory "$($artifact.name).json" + if (Test-Path $packageInfoFile) { + Copy-Item -Path $packageInfoFile -Destination $targetPackageInfoDirectory + } else { + $missingPackageInfo += $artifact.name + } +} + +# Downstream release steps (API review creation, release-completion marking, changelog verification) +# require a PackageInfo file per released package. Fail fast with an actionable list here rather than +# letting one of those later steps fail with a less obvious error. +if ($missingPackageInfo.Count -gt 0) { + throw "Missing PackageInfo JSON under '$sourcePackageInfoDirectory' for: $($missingPackageInfo -join ', ')." +} + +Write-Host "Auto-release signed artifacts staged at '$TargetDirectory'." \ No newline at end of file diff --git a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml new file mode 100644 index 000000000000..935a922656d2 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml @@ -0,0 +1,515 @@ +parameters: + - name: DependsOn + type: object + default: + - AutoReleasePrepare + - name: ServiceDirectory + type: string + - name: SDKType + type: string + - name: TestPipeline + type: boolean + default: false + - name: PublicPublishEnvironment + type: string + default: 'package-publish' + - name: PublicFeedUrl + type: string + default: 'maven.org' + +stages: + # Mirrors archetype-java-release-batch.yml for the post-merge auto-release scenario. + # The manual release template uses compile-time parameters.Artifacts loops, while this + # template uses runtime AutoReleaseArtifactsJson so only packages changed by the + # labeled merged PR are released after the shared Signing stage has completed. + - stage: JavaAutoRelease + displayName: Java auto-release + dependsOn: ${{ parameters.DependsOn }} + condition: >- + and( + succeeded(), + ne(variables['SetDevVersion'], 'true'), + ne(variables['Skip.Release'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-java-pr'), + eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.HasAutoReleaseArtifacts'], 'true') + ) + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - name: AutoReleaseArtifactsJson + value: $[ stageDependencies.AutoReleasePrepare.ResolveAutoReleasePackages.outputs['resolve.AutoReleaseArtifactsJson'] ] + jobs: + - job: PrepareAutoReleasePackages + displayName: Prepare auto-release packages + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - checkout: self + path: azure-sdk-for-java + - checkout: azure-sdk-build-tools + path: azure-sdk-build-tools + + - download: current + displayName: 'Download Artifact: packages-signed' + artifact: packages-signed + + - task: PowerShell@2 + displayName: Filter signed packages for auto-release + inputs: + pwsh: true + filePath: $(Pipeline.Workspace)/azure-sdk-for-java/eng/pipelines/scripts/Stage-AutoReleaseSignedArtifacts.ps1 + arguments: > + -SourceDirectory "$(Pipeline.Workspace)/packages-signed" + -TargetDirectory "$(Pipeline.Workspace)/packages-signed-auto-release" + -ArtifactsJson '$(AutoReleaseArtifactsJson)' + + - template: /eng/pipelines/templates/steps/maven-authenticate.yml + parameters: + SourceDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + + - template: tools/gpg/gpg.yml@azure-sdk-build-tools + - template: /eng/pipelines/templates/steps/gpg-sign-and-flatten.yml + parameters: + ArtifactDirectory: $(Pipeline.Workspace)/packages-signed-auto-release + OutputDirectory: $(Pipeline.Workspace)/packages-esrp-gpg-signed-auto-release + FlattenedESRPDirectory: $(Pipeline.Workspace)/packages-esrp-flattened-auto-release + JavaRepoRoot: $(Pipeline.Workspace)/azure-sdk-for-java + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Pipeline.Workspace)/packages-signed-auto-release' + ArtifactName: 'packages-signed-auto-release' + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Pipeline.Workspace)/packages-esrp-flattened-auto-release' + ArtifactName: 'packages-esrp-flattened-auto-release' + + - job: VerifyReleaseVersion + displayName: Verify auto-release versions + dependsOn: PrepareAutoReleasePackages + condition: ne(variables['Skip.VersionVerification'], 'true') + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - 'sdk/${{ parameters.ServiceDirectory }}' + - '**/*.xml' + - '!sdk/**/test-recordings' + - '!sdk/**/session-records' + + - download: current + displayName: 'Download Artifact: packages-signed-auto-release' + artifact: packages-signed-auto-release + + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + + - pwsh: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + $artifacts | Format-Table -Property groupId, name | Out-String | Write-Host + python --version + foreach ($artifact in $artifacts) { + python eng/versioning/set_versions.py --verify-version --artifact-id $artifact.name --group-id $artifact.groupId + } + displayName: Verify artifact versions for auto-release + + - pwsh: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json | Where-Object { -not $_.uberJar } + $artifacts | Format-Table -Property groupId, name | Out-String | Write-Host + foreach ($artifact in $artifacts) { + $(Build.SourcesDirectory)/eng/versioning/scan_for_unreleased_dependencies.ps1 $artifact.groupId $artifact.name $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + } + displayName: Verify no unreleased dependencies or beta dependencies for GA libraries + + - ${{ if ne(parameters.SDKType, 'data') }}: + - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml + parameters: + PackagePropertiesFolder: $(Pipeline.Workspace)/packages-signed-auto-release/PackageInfo + ForRelease: true + + - pwsh: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + foreach ($artifact in $artifacts) { + Write-Host "Verifying REST API spec location for $($artifact.name)." + eng/common/scripts/Verify-RestApiSpecLocation.ps1 ` + -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -PackageName $artifact.name ` + -ArtifactLocation "$(Pipeline.Workspace)/packages-signed-auto-release" + } + displayName: Verify REST API spec locations + env: + GH_TOKEN: $(azuresdk-github-pat) + condition: and(succeededOrFailed(), ne(variables['Skip.Verify-RestApiSpecLocation'], 'true')) + + - task: PowerShell@2 + displayName: Verify auto-release set + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/versioning/verify_release_set.ps1 + arguments: > + -ServiceDirectory $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + -ArtifactsList ('$(AutoReleaseArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, uberJar) + + - job: TagRepository + displayName: Create auto-release tags + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + dependsOn: VerifyReleaseVersion + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - template: /eng/common/pipelines/templates/steps/retain-run.yml + - template: /eng/common/pipelines/templates/steps/login-to-github.yml + parameters: + TokenOwners: + - $(Build.Repository.Name) + + - download: current + displayName: 'Download Artifact: packages-signed-auto-release' + artifact: packages-signed-auto-release + + - pwsh: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + foreach ($artifact in $artifacts) { + $artifactLocation = "$(Pipeline.Workspace)/packages-signed-auto-release/$($artifact.groupId)/$($artifact.name)" + Write-Host "Creating release tag for $($artifact.groupId):$($artifact.name)." + eng/common/scripts/create-tags-and-git-release.ps1 ` + -artifactLocation $artifactLocation ` + -packageRepository Maven ` + -releaseSha $(Build.SourceVersion) ` + -repoId $(Build.Repository.Name) + } + displayName: Verify package tags and create GitHub releases + env: + GH_TOKEN: $(GH_TOKEN) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - ${{ if eq(parameters.PublicFeedUrl, 'maven.org') }}: + - deployment: PublishESRPPackage + displayName: Publish auto-release packages to Maven Central via ESRP + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + timeoutInMinutes: 120 + environment: ${{ parameters.PublicPublishEnvironment }} + dependsOn: TagRepository + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + artifactName: 'packages-esrp-flattened-auto-release' + targetPath: '$(Pipeline.Workspace)/packages-esrp-flattened-auto-release' + pool: + name: azsdk-pool + image: windows-2022 + os: windows + strategy: + runOnce: + deploy: + steps: + - template: /eng/pipelines/templates/steps/java-esrp-publishing.yml + parameters: + FlattenedDirectory: $(Pipeline.Workspace)/packages-esrp-flattened-auto-release + + - job: PublishDevFeedPackage + displayName: Publish auto-release packages to Java Dev feed + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + timeoutInMinutes: 120 + ${{ if eq(parameters.PublicFeedUrl, 'maven.org') }}: + dependsOn: PublishESRPPackage + ${{ else }}: + dependsOn: TagRepository + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - checkout: azure-sdk-build-tools + path: azure-sdk-build-tools + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Repositories: + - Name: Azure/azure-sdk-for-java + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + + - download: current + displayName: 'Download Artifact: packages-signed-auto-release' + artifact: packages-signed-auto-release + + - template: /eng/pipelines/templates/steps/maven-authenticate.yml + parameters: + SourceDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - template: tools/gpg/gpg.yml@azure-sdk-build-tools + + - ${{ if ne(parameters.PublicFeedUrl, 'maven.org') }}: + - template: /eng/pipelines/templates/steps/java-dev-feed-publishing.yml + parameters: + ArtifactDirectory: $(Pipeline.Workspace)/packages-signed-auto-release + RepositoryUrl: ${{ parameters.PublicFeedUrl }} + JavaRepoRoot: $(Pipeline.Workspace)/azure-sdk-for-java + + - template: /eng/pipelines/templates/steps/java-dev-feed-publishing.yml + parameters: + ArtifactDirectory: $(Pipeline.Workspace)/packages-signed-auto-release + JavaRepoRoot: $(Pipeline.Workspace)/azure-sdk-for-java + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactName: packages-javadevfeed-auto-release-$(System.JobAttempt) + ArtifactPath: $(Pipeline.Workspace)/packages-signed-auto-release + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + + - task: AzureCLI@2 + displayName: Create and validate API reviews + condition: >- + and( + succeededOrFailed(), + not( + and( + eq(variables['Skip.CreateApiReview'], 'true'), + in( + lower(variables['Build.RequestedForEmail']), + 'bebroder@microsoft.com', + 'mharder@microsoft.com', + 'djurek@microsoft.com', + 'chononiw@microsoft.com', + 'raychen@microsoft.com', + 'trpresco@microsoft.com', + 'prmarott@microsoft.com' + ) + ) + ) + ) + inputs: + azureSubscription: 'APIView prod deployment' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + $artifactList = @($artifacts | Select-Object Name) + $packageInfoFiles = @($artifacts | ForEach-Object { "$(Pipeline.Workspace)/packages-signed-auto-release/PackageInfo/$($_.name).json" }) + $(Pipeline.Workspace)/azure-sdk-for-java/eng/common/scripts/Create-APIReview.ps1 ` + -PackageInfoFiles $packageInfoFiles ` + -ArtifactList $artifactList ` + -ArtifactPath '$(Pipeline.Workspace)/packages-signed-auto-release' ` + -ArtifactName packages-signed-auto-release ` + -SourceBranch '$(Build.SourceBranchName)' ` + -DefaultBranch '$(DefaultBranch)' ` + -ConfigFileDir '$(Pipeline.Workspace)/packages-signed-auto-release/PackageInfo' ` + -BuildId '$(Build.BuildId)' ` + -RepoName '$(Build.Repository.Name)' ` + -MarkPackageAsShipped $true + + - template: /eng/common/pipelines/templates/steps/install-azsdk-cli.yml + parameters: + SourceRootPath: $(Pipeline.Workspace)/azure-sdk-for-java + + - task: AzureCLI@2 + displayName: Mark auto-release packages as released + continueOnError: true + condition: and(succeeded(), ne(variables['Skip.MarkReleaseCompletion'], 'true')) + inputs: + azureSubscription: opensource-api-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + foreach ($artifact in $artifacts) { + $(Pipeline.Workspace)/azure-sdk-for-java/eng/common/scripts/Mark-ReleasePlanCompletion.ps1 ` + -PackageInfoFilePath "$(Pipeline.Workspace)/packages-signed-auto-release/PackageInfo/$($artifact.name).json" ` + -AzsdkExePath '$(AZSDK)' + } + workingDirectory: $(Pipeline.Workspace) + + - job: UpdatePackageVersion + displayName: Update auto-release package versions + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + dependsOn: PublishDevFeedPackage + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - download: none + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - '${{ parameters.ServiceDirectory }}' + - '**/*.xml' + - '**/*.md' + - '!sdk/**/test-recordings' + - '!sdk/**/session-records' + + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + + - pwsh: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + $artifacts | Format-Table -Property groupId, name | Out-String | Write-Host + python --version + foreach ($artifact in $artifacts) { + python eng/versioning/set_versions.py --increment-version --artifact-id $artifact.name --group-id $artifact.groupId + } + displayName: Increment auto-release package versions + + - script: | + python --version + python eng/versioning/update_versions.py --auto-version-increment --skip-readme + displayName: Update pom files for incremented versions + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-auto-release-$(Build.BuildId) + CommitMsg: "Increment package versions for ${{ parameters.ServiceDirectory }} auto-releases" + PRTitle: "Increment versions for ${{ parameters.ServiceDirectory }} auto-releases" + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + AuthToken: '' + + - ${{ if eq(parameters.PublicFeedUrl, 'maven.org') }}: + - job: PublishDocsMs + displayName: Docs.MS auto-release + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-java-pr')) + dependsOn: PublishDevFeedPackage + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/${{ parameters.ServiceDirectory }}/**/*.md + - '!**/pom*.xml' + + - download: current + displayName: 'Download Artifact: packages-signed-auto-release' + artifact: packages-signed-auto-release + + - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml + + - task: MavenAuthenticate@0 + displayName: 'Maven Authenticate' + inputs: + artifactsFeeds: 'azure-sdk-for-java' + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/common/pipelines/templates/steps/enable-long-path-support.yml + + - pwsh: | + Write-Host "##vso[task.setvariable variable=DocRepoLocation]$(System.DefaultWorkingDirectory)/doc" + displayName: Set $(DocRepoLocation) + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Repositories: + - Name: $(DocRepoOwner)/$(DocRepoName) + WorkingDirectory: $(DocRepoLocation) + Paths: + - docs-ref-services/ + - metadata/ + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(DocRepoLocation) + DefaultBranchVariableName: TargetBranchName + + - pwsh: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + $packageInfoLocations = @( + $artifacts | + Where-Object { -not ($_.PSObject.Properties.Name -contains 'skipPublishDocMs' -and "$($_.skipPublishDocMs)".ToLowerInvariant() -eq 'true') } | + ForEach-Object { "$(Pipeline.Workspace)/packages-signed-auto-release/PackageInfo/$($_.name).json" } + ) + + if ($packageInfoLocations.Count -eq 0) { + Write-Host "No auto-release packages require Docs.MS metadata updates." + return + } + + eng/common/scripts/Update-DocsMsMetadata.ps1 ` + -PackageInfoJsonLocations $packageInfoLocations ` + -DocRepoLocation "$(DocRepoLocation)" ` + -Language 'java' ` + -RepoId '$(Build.Repository.Name)' + displayName: Apply documentation updates + + - template: /eng/common/pipelines/templates/steps/login-to-github.yml + parameters: + TokenOwners: + - $(DocRepoOwner) + ScriptDirectory: eng/common/scripts + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(TargetBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update docs metadata" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts + AuthToken: $(GH_TOKEN) + + - job: PublishDocs + displayName: Publish auto-release docs to GitHubIO Blob Storage + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-java-pr')) + dependsOn: PublishDevFeedPackage + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + displayName: 'Download Artifact: packages-signed-auto-release' + artifact: packages-signed-auto-release + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + + - task: AzurePowerShell@5 + displayName: Copy auto-release package docs from signed artifacts to blob storage + inputs: + azureSubscription: 'Azure SDK Artifacts' + ScriptType: 'InlineScript' + azurePowerShellVersion: latestVersion + pwsh: true + Inline: | + $artifacts = '$(AutoReleaseArtifactsJson)' | ConvertFrom-Json + foreach ($artifact in $artifacts) { + if ($artifact.PSObject.Properties.Name -contains 'skipPublishDocGithubIo' -and "$($artifact.skipPublishDocGithubIo)".ToLowerInvariant() -eq 'true') { + Write-Host "Skipping GitHubIO docs for $($artifact.name)." + continue + } + + # Mirrors publish-blobs.yml: upload docs from the signed package artifact directory. + $artifactLocation = "$(Pipeline.Workspace)/packages-signed-auto-release/$($artifact.groupId)/$($artifact.name)" + Get-ChildItem -Recurse $artifactLocation + eng/common/scripts/copy-docs-to-blobstorage.ps1 ` + -DocLocation $artifactLocation ` + -BlobAccountName "https://azuresdkdocs.blob.core.windows.net" ` + -PublicArtifactLocation $artifactLocation ` + -RepoReplaceRegex "(https://github.com/$(Build.Repository.Name)/(?:blob|tree)/)$(DefaultBranch)" + } + env: + AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' + SYSTEM_ACCESSTOKEN: $(System.AccessToken) \ No newline at end of file diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 2f3a628459bd..6406f33788e1 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -191,3 +191,21 @@ extends: PublicFeedUrl: ${{ parameters.PublicFeedUrl }} PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main'), not(contains(variables['Build.DefinitionName'], 'tests-weekly'))) }}: + - template: /eng/common/pipelines/templates/stages/archetype-auto-release-prepare.yml + parameters: + DependsOn: + # Signing is produced by archetype-java-release-batch.yml above. Auto-release prepares after signing + # so the manual release path remains unchanged while auto-release filters the signed output. + - Signing + Artifacts: ${{ parameters.Artifacts }} + - template: archetype-java-auto-release-batch.yml + parameters: + DependsOn: + - AutoReleasePrepare + ServiceDirectory: ${{ parameters.ServiceDirectory }} + SDKType: ${{ parameters.SDKType }} + TestPipeline: ${{ parameters.TestPipeline }} + PublicFeedUrl: ${{ parameters.PublicFeedUrl }} + PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + diff --git a/sdk/keyvault/azure-security-keyvault-jca/CHANGELOG.md b/sdk/keyvault/azure-security-keyvault-jca/CHANGELOG.md index 034766972361..fbfbab4dae26 100644 --- a/sdk/keyvault/azure-security-keyvault-jca/CHANGELOG.md +++ b/sdk/keyvault/azure-security-keyvault-jca/CHANGELOG.md @@ -7,6 +7,7 @@ ### Breaking Changes ### Bugs Fixed +- Fixed an issue where a disabled certificate in Azure Key Vault caused keystore initialization to fail with an HTTP 403 error. Disabled certificates are now skipped when loading aliases and a warning is logged for each skipped certificate. [#49730](https://github.com/Azure/azure-sdk-for-java/pull/49730) ### Other Changes diff --git a/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/KeyVaultClient.java b/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/KeyVaultClient.java index 6fadbad95419..411f6b1db75f 100644 --- a/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/KeyVaultClient.java +++ b/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/KeyVaultClient.java @@ -288,6 +288,14 @@ public List getAliases() { for (CertificateItem certificateItem : certificateListResult.getValue()) { String id = certificateItem.getId(); String alias = getCertificateNameFromCertificateItemId(id); + + // Skip certificates that are explicitly disabled in Key Vault. Attempting to load a disabled + // certificate's key/secret later would fail with an HTTP 403 and break keystore initialization. + if (!certificateItem.isEnabled()) { + LOGGER.log(WARNING, "Skipping disabled certificate with alias: {0}", alias); + continue; + } + result.add(alias); } } else { diff --git a/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItem.java b/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItem.java index 9878b5829999..8bd04d35a566 100644 --- a/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItem.java +++ b/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItem.java @@ -19,6 +19,11 @@ public class CertificateItem implements JsonSerializable { */ private String id; + /** + * Stores the management attributes of the certificate. + */ + private CertificateItemAttributes attributes; + /** * Get the id. * @@ -37,10 +42,42 @@ public void setId(String id) { this.id = id; } + /** + * Get the management attributes of the certificate. + * + * @return the management attributes of the certificate, or {@code null} if not specified. + */ + public CertificateItemAttributes getAttributes() { + return attributes; + } + + /** + * Set the management attributes of the certificate. + * + * @param attributes the management attributes of the certificate. + */ + public void setAttributes(CertificateItemAttributes attributes) { + this.attributes = attributes; + } + + /** + * Indicates whether the certificate is enabled. + *

+ * A certificate is considered enabled unless its attributes explicitly mark it as disabled (i.e. + * {@code attributes.enabled == false}). When the attributes or the {@code enabled} flag are absent, the certificate + * is treated as enabled for backward compatibility. + * + * @return {@code false} only when the certificate is explicitly disabled, otherwise {@code true}. + */ + public boolean isEnabled() { + return attributes == null || !Boolean.FALSE.equals(attributes.isEnabled()); + } + @Override public JsonWriter toJson(JsonWriter jsonWriter) throws IOException { jsonWriter.writeStartObject(); jsonWriter.writeStringField("id", this.id); + jsonWriter.writeJsonField("attributes", this.attributes); return jsonWriter.writeEndObject(); } @@ -66,6 +103,8 @@ public static CertificateItem fromJson(JsonReader jsonReader) throws IOException if ("id".equals(fieldName)) { deserializedCertificateItem.id = reader.getString(); + } else if ("attributes".equals(fieldName)) { + deserializedCertificateItem.attributes = CertificateItemAttributes.fromJson(reader); } else { reader.skipChildren(); } diff --git a/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItemAttributes.java b/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItemAttributes.java new file mode 100644 index 000000000000..691887af4a5c --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-jca/src/main/java/com/azure/security/keyvault/jca/implementation/model/CertificateItemAttributes.java @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.security.keyvault.jca.implementation.model; + +import com.azure.json.JsonReader; +import com.azure.json.JsonSerializable; +import com.azure.json.JsonToken; +import com.azure.json.JsonWriter; + +import java.io.IOException; + +/** + * The management attributes of a {@link CertificateItem}, as returned by the Azure Key Vault "list certificates" REST + * API. + */ +public class CertificateItemAttributes implements JsonSerializable { + /** + * Stores whether the certificate is enabled. + */ + private Boolean enabled; + + /** + * Get whether the certificate is enabled. + * + * @return whether the certificate is enabled, or {@code null} if the attribute was not specified. + */ + public Boolean isEnabled() { + return enabled; + } + + /** + * Set whether the certificate is enabled. + * + * @param enabled whether the certificate is enabled. + */ + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + @Override + public JsonWriter toJson(JsonWriter jsonWriter) throws IOException { + jsonWriter.writeStartObject(); + jsonWriter.writeBooleanField("enabled", this.enabled); + + return jsonWriter.writeEndObject(); + } + + /** + * Reads an instance of {@link CertificateItemAttributes} from the {@link JsonReader}. + * + * @param jsonReader The {@link JsonReader} being read. + * + * @return An instance of {@link CertificateItemAttributes} if the {@link JsonReader} was pointing to an instance of + * it, or {@code null} if it was pointing to JSON {@code null}. + * + * @throws IOException If an error occurs while reading the {@link CertificateItemAttributes}. + */ + public static CertificateItemAttributes fromJson(JsonReader jsonReader) throws IOException { + return jsonReader.readObject(reader -> { + CertificateItemAttributes deserializedCertificateItemAttributes = new CertificateItemAttributes(); + + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + + reader.nextToken(); + + if ("enabled".equals(fieldName)) { + deserializedCertificateItemAttributes.enabled = reader.getNullable(JsonReader::getBoolean); + } else { + reader.skipChildren(); + } + } + + return deserializedCertificateItemAttributes; + }); + } +} diff --git a/sdk/keyvault/azure-security-keyvault-jca/src/test/java/com/azure/security/keyvault/jca/implementation/KeyVaultClientTest.java b/sdk/keyvault/azure-security-keyvault-jca/src/test/java/com/azure/security/keyvault/jca/implementation/KeyVaultClientTest.java index 22f80900efa6..19f7631bf4e6 100644 --- a/sdk/keyvault/azure-security-keyvault-jca/src/test/java/com/azure/security/keyvault/jca/implementation/KeyVaultClientTest.java +++ b/sdk/keyvault/azure-security-keyvault-jca/src/test/java/com/azure/security/keyvault/jca/implementation/KeyVaultClientTest.java @@ -6,6 +6,7 @@ import com.azure.security.keyvault.jca.PropertyConvertorUtils; import com.azure.security.keyvault.jca.implementation.model.AccessToken; import com.azure.security.keyvault.jca.implementation.model.CertificateItem; +import com.azure.security.keyvault.jca.implementation.model.CertificateItemAttributes; import com.azure.security.keyvault.jca.implementation.model.CertificateListResult; import com.azure.security.keyvault.jca.implementation.utils.AccessTokenUtil; import com.azure.security.keyvault.jca.implementation.utils.HttpUtil; @@ -19,6 +20,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyMap; @@ -109,6 +111,97 @@ public void testGetAliasWithCertificateInfoWith2Pages() { } } + @Test + public void testGetAliasFiltersOutDisabledCertificate() { + try (MockedStatic utilities = Mockito.mockStatic(HttpUtil.class)) { + utilities.when(() -> HttpUtil.validateUri(anyString(), anyString())).thenCallRealMethod(); + utilities.when(() -> HttpUtil.addTrailingSlashIfRequired(anyString())).thenCallRealMethod(); + + // Enabled certificate. + CertificateItemAttributes enabledAttributes = new CertificateItemAttributes(); + enabledAttributes.setEnabled(true); + CertificateItem enabledCertificate = new CertificateItem(); + enabledCertificate.setId("certificates/client-cert-active"); + enabledCertificate.setAttributes(enabledAttributes); + + // Disabled certificate. This one previously caused an HTTP 403 while initializing the keystore. + CertificateItemAttributes disabledAttributes = new CertificateItemAttributes(); + disabledAttributes.setEnabled(false); + CertificateItem disabledCertificate = new CertificateItem(); + disabledCertificate.setId("certificates/client-cert-unused"); + disabledCertificate.setAttributes(disabledAttributes); + + CertificateListResult certificateListResult = new CertificateListResult(); + certificateListResult.setValue(Arrays.asList(enabledCertificate, disabledCertificate)); + + String certificateListResultString = JsonConverterUtil.toJson(certificateListResult); + utilities.when(() -> HttpUtil.get(notNull(), anyMap())).thenReturn(certificateListResultString); + + KeyVaultClient keyVaultClient = new KeyVaultClient(KEY_VAULT_TEST_URI_GLOBAL, null); + List result = keyVaultClient.getAliases(); + + assertEquals(1, result.size()); + assertTrue(result.contains("client-cert-active")); + assertFalse(result.contains("client-cert-unused")); + } + } + + @Test + public void testGetAliasKeepsEnabledAndAttributelessCertificates() { + try (MockedStatic utilities = Mockito.mockStatic(HttpUtil.class)) { + utilities.when(() -> HttpUtil.validateUri(anyString(), anyString())).thenCallRealMethod(); + utilities.when(() -> HttpUtil.addTrailingSlashIfRequired(anyString())).thenCallRealMethod(); + + // Certificate explicitly enabled. + CertificateItemAttributes enabledAttributes = new CertificateItemAttributes(); + enabledAttributes.setEnabled(true); + CertificateItem enabledCertificate = new CertificateItem(); + enabledCertificate.setId("certificates/enabledCertificate"); + enabledCertificate.setAttributes(enabledAttributes); + + // Certificate without attributes, which must be treated as enabled for backward compatibility. + CertificateItem attributelessCertificate = new CertificateItem(); + attributelessCertificate.setId("certificates/attributelessCertificate"); + + CertificateListResult certificateListResult = new CertificateListResult(); + certificateListResult.setValue(Arrays.asList(enabledCertificate, attributelessCertificate)); + + String certificateListResultString = JsonConverterUtil.toJson(certificateListResult); + utilities.when(() -> HttpUtil.get(notNull(), anyMap())).thenReturn(certificateListResultString); + + KeyVaultClient keyVaultClient = new KeyVaultClient(KEY_VAULT_TEST_URI_GLOBAL, null); + List result = keyVaultClient.getAliases(); + + assertEquals(2, result.size()); + assertTrue(result.containsAll(Arrays.asList("enabledCertificate", "attributelessCertificate"))); + } + } + + @Test + public void testGetAliasFiltersDisabledCertificateFromRawResponse() { + try (MockedStatic utilities = Mockito.mockStatic(HttpUtil.class)) { + utilities.when(() -> HttpUtil.validateUri(anyString(), anyString())).thenCallRealMethod(); + utilities.when(() -> HttpUtil.addTrailingSlashIfRequired(anyString())).thenCallRealMethod(); + + // A response that mirrors the shape returned by the Azure Key Vault "list certificates" REST API, with one + // enabled and one disabled certificate. + String rawResponse = "{\"value\":[" + + "{\"id\":\"https://fake.vault.azure.net/certificates/client-cert-active\"," + + "\"attributes\":{\"enabled\":true,\"nbf\":1783324860,\"exp\":1814861460}}," + + "{\"id\":\"https://fake.vault.azure.net/certificates/client-cert-unused\"," + + "\"attributes\":{\"enabled\":false,\"nbf\":1783324860,\"exp\":1814861460}}]," + "\"nextLink\":null}"; + + utilities.when(() -> HttpUtil.get(notNull(), anyMap())).thenReturn(rawResponse); + + KeyVaultClient keyVaultClient = new KeyVaultClient(KEY_VAULT_TEST_URI_GLOBAL, null); + List result = keyVaultClient.getAliases(); + + assertEquals(1, result.size()); + assertTrue(result.contains("client-cert-active")); + assertFalse(result.contains("client-cert-unused")); + } + } + @Test public void testCacheToken() { try (MockedStatic tokenUtilMockedStatic = Mockito.mockStatic(AccessTokenUtil.class);