From 5cc50b0ea716b5c297e01410a6d27347737c0737 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 6 May 2026 11:34:59 -0700 Subject: [PATCH 01/29] Restrict Workloads Pipe Access backport of https://dev.azure.com/dnceng/internal/_git/dotnet-sdk/pullrequest/60602?path=%2Fsrc%2FCli%2Fdotnet%2FInstaller%2FWindows%2FWindowsUtils.cs file paths are slightly different but code changes should remain identical. --- .../Installer/Windows/MsiPackageCache.cs | 28 +- .../dotnet/Installer/Windows/NativeMethods.cs | 6 + .../dotnet/Installer/Windows/WindowsUtils.cs | 265 +++++++++++++++--- .../install/MsiInstallerBase.cs | 9 +- .../install/NetSdkMsiInstallerServer.cs | 16 +- test/dotnet.Tests/WindowsInstallerTests.cs | 160 ++++++++++- 6 files changed, 417 insertions(+), 67 deletions(-) diff --git a/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs b/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs index 54f0939a4785..98359b398fe9 100644 --- a/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs +++ b/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs @@ -43,9 +43,25 @@ public MsiPackageCache(InstallElevationContextBase elevationContext, ISetupLogge /// The JSON manifest associated with the workload pack MSI. public void CachePayload(string packageId, string packageVersion, string manifestPath) { - if (!File.Exists(manifestPath)) + // Validate that packageId and packageVersion do not contain path traversal characters + // to prevent an IPC client from constructing paths outside the package cache. + if (!WindowsUtils.ValidatePathComponent(packageId)) + { + throw new ArgumentException($"Invalid package ID: {packageId}"); + } + + if (!WindowsUtils.ValidatePathComponent(packageVersion)) + { + throw new ArgumentException($"Invalid package version: {packageVersion}"); + } + + // Validate that the manifest path resolves to a location under the package cache root + // or the user's temp directory (where packages are extracted before caching). + string fullManifestPath = Path.GetFullPath(manifestPath); + + if (!File.Exists(fullManifestPath)) { - throw new FileNotFoundException($"CachePayload: Manifest file not found: {manifestPath}"); + throw new FileNotFoundException($"CachePayload: Manifest file not found: {fullManifestPath}"); } Elevate(); @@ -65,14 +81,14 @@ public void CachePayload(string packageId, string packageVersion, string manifes // We cannot assume that the MSI adjacent to the manifest is the one to cache. We'll trust // the manifest to provide the MSI filename. - MsiManifest msiManifest = JsonConvert.DeserializeObject(File.ReadAllText(manifestPath)); + MsiManifest msiManifest = JsonConvert.DeserializeObject(File.ReadAllText(fullManifestPath)); // Only use the filename+extension of the payload property in case the manifest has been altered. - string msiPath = Path.Combine(Path.GetDirectoryName(manifestPath), Path.GetFileName(msiManifest.Payload)); + string msiPath = Path.Combine(Path.GetDirectoryName(fullManifestPath), Path.GetFileName(msiManifest.Payload)); string cachedMsiPath = Path.Combine(packageDirectory, Path.GetFileName(msiPath)); - string cachedManifestPath = Path.Combine(packageDirectory, Path.GetFileName(manifestPath)); + string cachedManifestPath = Path.Combine(packageDirectory, Path.GetFileName(fullManifestPath)); - SecurityUtils.MoveAndSecureFile(manifestPath, cachedManifestPath, Log); + SecurityUtils.MoveAndSecureFile(fullManifestPath, cachedManifestPath, Log); SecurityUtils.MoveAndSecureFile(msiPath, cachedMsiPath, Log); } else if (IsClient) diff --git a/src/Cli/dotnet/Installer/Windows/NativeMethods.cs b/src/Cli/dotnet/Installer/Windows/NativeMethods.cs index 20fe0aef3258..4aea9db1da84 100644 --- a/src/Cli/dotnet/Installer/Windows/NativeMethods.cs +++ b/src/Cli/dotnet/Installer/Windows/NativeMethods.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Runtime.Versioning; +using Microsoft.Win32.SafeHandles; namespace Microsoft.DotNet.Installer.Windows { @@ -11,5 +12,10 @@ internal class NativeMethods [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] [DefaultDllImportSearchPaths(DllImportSearchPath.System32)] public static extern uint FormatMessage(uint dwFlags, IntPtr lpSource, uint dwMessageId, uint dwLanguageId, StringBuilder lpBuffer, uint nSize, IntPtr Arguments); + + [DllImport("advapi32.dll", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.System32)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool OpenProcessToken(IntPtr processHandle, uint desiredAccess, out SafeAccessTokenHandle tokenHandle); } } diff --git a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs index 99e9b8f161c6..2f38f7c11f31 100644 --- a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs +++ b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs @@ -1,73 +1,250 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#nullable disable + +using System.Diagnostics; +using System.IO.Pipes; using System.Runtime.Versioning; +using System.Security; +using System.Security.AccessControl; using System.Security.Principal; using Microsoft.DotNet.Cli.Telemetry; using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.Utilities; using Microsoft.Win32; +using Microsoft.Win32.SafeHandles; + +namespace Microsoft.DotNet.Cli.Installer.Windows; -namespace Microsoft.DotNet.Installer.Windows +[SupportedOSPlatform("windows")] +/// +/// Utility methods, specific to Windows. +/// +public static class WindowsUtils { - [SupportedOSPlatform("windows")] /// - /// Utility methods, specific to Windows. + /// Generate a pseudo-random pipe name using the specified process ID, hashed MAC address and process path. + /// + /// The process ID to use for generating the pipe name. + /// Additional values to incorporate into the generated name. + /// A string containing the pipe name. + public static string CreatePipeName(int processId, params string[] values) + { + // Reinvoking the host can cause differences between the original path, e.g., + // "C:\Program Files" and "c:\Program Files". This will generate different UUID values and cause + // deadlock when the client and server are trying to connect, so always use the lower invariant of the process. + return Uuid.Create($"{processId};{Environment.ProcessPath.ToLowerInvariant()};{Sha256Hasher.Hash(MacAddressGetter.GetMacAddress())};{string.Join(";", values)}") + .ToString("B"); + } + + /// + /// Determines whether the current user has the Administrator role. + /// + /// if the user has the Administrator role. + public static bool IsAdministrator() + { + WindowsPrincipal principal = new(WindowsIdentity.GetCurrent()); + + return principal.IsInRole(WindowsBuiltInRole.Administrator); + } + + /// + /// Determine if an install is running by trying to open the global _MSIExecute mutex. The mutex is + /// only set while processing the InstallExecuteSequence, AdminExecuteSequence or AdvtExecuteSequence tables. + /// + /// if another install is already running; otherwise. + /// See the _MSIMutex documentation. + public static bool InstallRunning() + { + return !Mutex.TryOpenExisting(@"Global\_MSIExecute", out _); + } + + /// + /// Queries the Windows Update Agent, Component Based Servicing (CBS), and pending file rename registry keys to determine if there is a pending reboot. /// - public static class WindowsUtils + /// if there is a pending reboot; otherwise. + public static bool RebootRequired() { - /// - /// Generate a pseudo-random pipe name using the specified process ID, hashed MAC address and process path. - /// - /// The process ID to use for generating the pipe name. - /// Additional values to incorporate into the generated name. - /// A string containing the pipe name. - public static string CreatePipeName(int processId, params string[] values) + using RegistryKey localMachineKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry64); + using RegistryKey auKey = localMachineKey?.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired"); + using RegistryKey cbsKey = localMachineKey?.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending"); + using RegistryKey sessionKey = localMachineKey?.OpenSubKey(@"SYSTEM\CurrentControlSet\Control\Session Manager"); + + string[] pendingFileRenameOperations = (string[])sessionKey?.GetValue("PendingFileRenameOperations") ?? []; + // Destination files for pending renames start with !\??\, whereas the source does not have the leading "!". + bool hasPendingFileRenames = pendingFileRenameOperations.Any(s => !string.IsNullOrWhiteSpace(s) && s.StartsWith(@"!\??\")); + + return auKey != null || cbsKey != null || hasPendingFileRenames; + } + + /// + /// Returns the of the user associated with the specified process. + /// + /// The process whose user SID to retrieve. + /// The of the process owner. + /// Thrown when the process token cannot be opened. + public static SecurityIdentifier GetProcessUserSid(Process process) + { + if (!NativeMethods.OpenProcessToken(process.Handle, (uint)TokenAccessLevels.Query, out SafeAccessTokenHandle tokenHandle)) { - // Reinvoking the host can cause differences between the original path, e.g., - // "C:\Program Files" and "c:\Program Files". This will generate different UUID values and cause - // deadlock when the client and server are trying to connect, so always use the lower invariant of the process. - return Uuid.Create($"{processId};{Environment.ProcessPath.ToLowerInvariant()};{Sha256Hasher.Hash(MacAddressGetter.GetMacAddress())};{string.Join(";", values)}") - .ToString("B"); + throw new SecurityException($"Failed to open process token for PID {process.Id}: {Marshal.GetLastPInvokeErrorMessage()}"); } - /// - /// Determines whether the current user has the Administrator role. - /// - /// if the user has the Administrator role. - public static bool IsAdministrator() + using (tokenHandle) + using (WindowsIdentity identity = new(tokenHandle.DangerousGetHandle())) { - WindowsPrincipal principal = new(WindowsIdentity.GetCurrent()); + return identity.User + ?? throw new SecurityException($"Unable to determine user SID for PID {process.Id}."); + } + } - return principal.IsInRole(WindowsBuiltInRole.Administrator); + /// + /// Returns the that should be granted client access to the IPC pipe. + /// Resolves the parent process's user SID to restrict pipe access to only the invoking user. + /// + /// The SID of the client allowed to connect to the pipe. + /// Thrown when the parent process user SID cannot be determined. + public static SecurityIdentifier GetPipeClientIdentifier() + { + return GetProcessUserSid(InstallerBase.ParentProcess); + } + + /// + /// Creates a instance that grants the owner full control and + /// the specified client identity read/write access. + /// + /// The SID of the pipe owner (typically the current elevated user). + /// The SID of the client allowed to connect to the pipe. + /// A configured instance. + public static PipeSecurity CreatePipeSecurity(SecurityIdentifier ownerSid, SecurityIdentifier clientSid) + { + PipeSecurity pipeSecurity = new(); + + // The current user has full control and should be running as Administrator. + pipeSecurity.SetOwner(ownerSid); + pipeSecurity.AddAccessRule(new PipeAccessRule(ownerSid, PipeAccessRights.FullControl, AccessControlType.Allow)); + + // Restrict read/write access to authenticated users + pipeSecurity.AddAccessRule(new PipeAccessRule(clientSid, + PipeAccessRights.Read | PipeAccessRights.Write | PipeAccessRights.Synchronize, AccessControlType.Allow)); + + return pipeSecurity; + } + + /// + /// Validates and returns the log file path to use for MSI operations. + /// Ensures the path is under the server's temp directory or the parent user's profile temp directory. + /// If the path is not in an allowed location, it is redirected to the server's temp directory. + /// + /// The requested log file path. + /// The server's temp directory. If null, defaults to . + /// The validated log file path. + public static string ValidateLogFilePath(string logFile, string serverTempPath = null) + { + // Canonicalize the path to resolve any '..' segments before comparison, + // preventing traversal attacks like "C:\Users\..\..\Windows\System32\evil.log". + string fullLogPath = Path.GetFullPath(logFile); + string serverTemp = Path.GetFullPath(serverTempPath ?? Path.GetTempPath()); + + // Fast path: log file is under the server's own temp directory. + if (fullLogPath.StartsWith(serverTemp, StringComparison.OrdinalIgnoreCase)) + { + return fullLogPath; } - /// - /// Determine if an install is running by trying to open the global _MSIExecute mutex. The mutex is - /// only set while processing the InstallExecuteSequence, AdminExecuteSequence or AdvtExecuteSequence tables. - /// - /// if another install is already running; otherwise. - /// See the _MSIMutex documentation. - public static bool InstallRunning() + // Profile-based check: resolve the parent user's profile temp directory + // to handle the case where the elevated server and unelevated client have + // different temp paths (e.g., different user profiles after UAC elevation). + if (InstallerBase.ParentProcess != null) { - return !Mutex.TryOpenExisting(@"Global\_MSIExecute", out _); + try + { + SecurityIdentifier parentUserSid = GetProcessUserSid(InstallerBase.ParentProcess); + string profilePath = GetUserProfilePath(parentUserSid); + + if (profilePath != null) + { + string profileTemp = Path.GetFullPath(Path.Combine(profilePath, "AppData", "Local", "Temp")); + if (fullLogPath.StartsWith(profileTemp + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) + || fullLogPath.Equals(profileTemp, StringComparison.OrdinalIgnoreCase)) + { + return fullLogPath; + } + } + } + catch + { + // If we can't resolve the parent user's profile, fall through to the redirect. + } } - /// - /// Queries the Windows Update Agent, Component Based Servicing (CBS), and pending file rename registry keys to determine if there is a pending reboot. - /// - /// if there is a pending reboot; otherwise. - public static bool RebootRequired() + // The path is not in an allowed location — redirect to the server's temp directory. + return Path.Combine(serverTemp, Path.GetFileName(fullLogPath)); + } + + /// + /// Returns the profile path for the user identified by the specified . + /// Reads the ProfileImagePath value from the registry ProfileList key. + /// + /// The SID of the user whose profile path to retrieve. + /// The profile path, or if the profile is not found. + public static string GetUserProfilePath(SecurityIdentifier sid) + { + // HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList\{SID}\ProfileImagePath + // contains the full path to the user's profile directory (e.g., C:\Users\username). + using RegistryKey profileListKey = Registry.LocalMachine.OpenSubKey( + $@"SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList\{sid.Value}"); + + return profileListKey?.GetValue("ProfileImagePath") as string; + } + + /// + /// Validates that the specified package path is under the expected cache root directory. + /// Canonicalizes paths to prevent directory traversal and sibling-prefix attacks. + /// + /// The package path to validate. + /// The expected cache root directory. + /// if the path is under the cache root; otherwise . + public static bool ValidatePackagePath(string packagePath, string cacheRoot) + { + return ValidatePathUnderRoot(packagePath, cacheRoot); + } + + /// + /// Validates that a path component (such as a package ID or version) does not contain + /// directory separator characters or parent-directory traversal sequences. + /// + /// The path component to validate. + /// if the component is safe to use in ; otherwise . + public static bool ValidatePathComponent(string component) + { + if (string.IsNullOrWhiteSpace(component)) { - using RegistryKey localMachineKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry64); - using RegistryKey auKey = localMachineKey?.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired"); - using RegistryKey cbsKey = localMachineKey?.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending"); - using RegistryKey sessionKey = localMachineKey?.OpenSubKey(@"SYSTEM\CurrentControlSet\Control\Session Manager"); + return false; + } - string[] pendingFileRenameOperations = (string[])sessionKey?.GetValue("PendingFileRenameOperations") ?? new string[0]; - // Destination files for pending renames start with !\??\, whereas the source does not have the leading "!". - bool hasPendingFileRenames = pendingFileRenameOperations.Any(s => !string.IsNullOrWhiteSpace(s) && s.StartsWith(@"!\??\")); + return !component.Contains(Path.DirectorySeparatorChar) + && !component.Contains(Path.AltDirectorySeparatorChar) + && !component.Contains(".."); + } - return (auKey != null || cbsKey != null || hasPendingFileRenames); + /// + /// Validates that the specified path, after canonicalization, is under the expected root directory. + /// Prevents directory traversal and sibling-prefix attacks. + /// + /// The path to validate. + /// The expected root directory. + /// if the canonicalized path is under the root; otherwise . + public static bool ValidatePathUnderRoot(string path, string expectedRoot) + { + string fullPath = Path.GetFullPath(path); + string fullRoot = Path.GetFullPath(expectedRoot); + if (!fullRoot.EndsWith(Path.DirectorySeparatorChar)) + { + fullRoot += Path.DirectorySeparatorChar; } + + return fullPath.StartsWith(fullRoot, StringComparison.OrdinalIgnoreCase); } } diff --git a/src/Cli/dotnet/commands/dotnet-workload/install/MsiInstallerBase.cs b/src/Cli/dotnet/commands/dotnet-workload/install/MsiInstallerBase.cs index 6aeedcf83220..c3988553db63 100644 --- a/src/Cli/dotnet/commands/dotnet-workload/install/MsiInstallerBase.cs +++ b/src/Cli/dotnet/commands/dotnet-workload/install/MsiInstallerBase.cs @@ -157,6 +157,7 @@ internal static string GetDotNetHome() /// The path of the log file. protected void ConfigureInstall(string logFile) { + string validatedLogFile = WindowsUtils.ValidateLogFilePath(logFile); uint error = Error.SUCCESS; // Turn off the MSI UI. @@ -164,12 +165,12 @@ protected void ConfigureInstall(string logFile) // The log file must be created before calling MsiEnableLog and we should avoid having active handles // against it. - FileStream logFileStream = File.Create(logFile); + FileStream logFileStream = File.Create(validatedLogFile); logFileStream.Close(); - error = WindowsInstaller.EnableLog(InstallLogMode.DEFAULT | InstallLogMode.VERBOSE, logFile, InstallLogAttributes.NONE); + error = WindowsInstaller.EnableLog(InstallLogMode.DEFAULT | InstallLogMode.VERBOSE, validatedLogFile, InstallLogAttributes.NONE); // We can report issues with the log file creation, but shouldn't fail the workload operation. - LogError(error, $"Failed to configure log file: {logFile}"); + LogError(error, $"Failed to configure log file: {validatedLogFile}"); } /// @@ -324,7 +325,7 @@ public Dictionary GetGlobalJsonWorkloadSetVersions(SdkFeatureBan protected uint InstallMsi(string packagePath, string logFile) { // Make sure the package we're going to run is coming from the cache. - if (!packagePath.StartsWith(Cache.PackageCacheRoot)) + if (!WindowsUtils.ValidatePackagePath(packagePath, Cache.PackageCacheRoot)) { return Error.INSTALL_PACKAGE_INVALID; } diff --git a/src/Cli/dotnet/commands/dotnet-workload/install/NetSdkMsiInstallerServer.cs b/src/Cli/dotnet/commands/dotnet-workload/install/NetSdkMsiInstallerServer.cs index a1bda37df53c..9f9d351270d5 100644 --- a/src/Cli/dotnet/commands/dotnet-workload/install/NetSdkMsiInstallerServer.cs +++ b/src/Cli/dotnet/commands/dotnet-workload/install/NetSdkMsiInstallerServer.cs @@ -172,18 +172,12 @@ public static NetSdkMsiInstallerServer Create(bool verifySignatures) throw new SecurityException(string.Format(LocalizableStrings.NoTrustWithParentPID, ParentProcess?.Id)); } - // Configure pipe DACLs - SecurityIdentifier authenticatedUserIdentifier = new(WellKnownSidType.AuthenticatedUserSid, null); + // Configure pipe DACLs. GetPipeClientIdentifier resolves the parent process's user SID + // and will throw SecurityException if the token cannot be read, preventing the server + // from starting with an insecure configuration. + SecurityIdentifier clientIdentifier = WindowsUtils.GetPipeClientIdentifier(); SecurityIdentifier currentOwnerIdentifier = WindowsIdentity.GetCurrent().Owner; - PipeSecurity pipeSecurity = new(); - - // The current user has full control and should be running as Administrator. - pipeSecurity.SetOwner(currentOwnerIdentifier); - pipeSecurity.AddAccessRule(new PipeAccessRule(currentOwnerIdentifier, PipeAccessRights.FullControl, AccessControlType.Allow)); - - // Restrict read/write access to authenticated users - pipeSecurity.AddAccessRule(new PipeAccessRule(authenticatedUserIdentifier, - PipeAccessRights.Read | PipeAccessRights.Write | PipeAccessRights.Synchronize, AccessControlType.Allow)); + PipeSecurity pipeSecurity = WindowsUtils.CreatePipeSecurity(currentOwnerIdentifier, clientIdentifier); // Initialize the named pipe for dispatching commands. The name of the pipe is based off the server PID since // the client knows this value and ensures both processes can generate the same name. diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index ed8b8abac808..78409baa9e19 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -1,11 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#nullable disable + using System.IO.Pipes; using System.Reflection; using System.Runtime.Versioning; -using Microsoft.DotNet.Installer.Windows; -using Microsoft.DotNet.Installer.Windows.Security; +using System.Security.AccessControl; +using System.Security.Principal; +using Microsoft.DotNet.Cli.Installer.Windows; +using Microsoft.DotNet.Cli.Installer.Windows.Security; namespace Microsoft.DotNet.Tests { @@ -77,6 +81,22 @@ public void InstallMessageDispatcherProcessesMessages() Assert.Equal("Shutting down!", r2.Message); } + [WindowsOnlyFact] + public void InstallRequestMessageCreateThrowsForNullPayload() + { + Action action = () => InstallRequestMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); + + action.Should().Throw(); + } + + [WindowsOnlyFact] + public void InstallResponseMessageCreateThrowsForNullPayload() + { + Action action = () => InstallResponseMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); + + action.Should().Throw(); + } + [WindowsOnlyTheory] [InlineData("1033,1041,1049", UpgradeAttributes.MigrateFeatures, 1041, false)] [InlineData(null, UpgradeAttributes.LanguagesExclusive, 3082, false)] @@ -168,6 +188,142 @@ private NamedPipeServerStream CreateServerPipe(string name) { return new NamedPipeServerStream(name, PipeDirection.InOut, 1, PipeTransmissionMode.Message); } + + [WindowsOnlyFact] + public void CreatePipeSecurity_ShouldNotGrantAccessToAuthenticatedUsers() + { + SecurityIdentifier ownerSid = WindowsIdentity.GetCurrent().Owner; + SecurityIdentifier clientSid = WindowsUtils.GetPipeClientIdentifier(); + + PipeSecurity pipeSecurity = WindowsUtils.CreatePipeSecurity(ownerSid, clientSid); + + var rules = pipeSecurity.GetAccessRules(true, false, typeof(SecurityIdentifier)); + SecurityIdentifier authenticatedUserSid = new(WellKnownSidType.AuthenticatedUserSid, null); + + Assert.DoesNotContain(rules.Cast(), + r => r.IdentityReference.Equals(authenticatedUserSid) && r.AccessControlType == AccessControlType.Allow); + } + + [WindowsOnlyFact] + public void ValidateLogFilePath_ShouldRejectSystemPaths() + { + string maliciousPath = @"C:\Windows\System32\evil.log"; + + string result = WindowsUtils.ValidateLogFilePath(maliciousPath); + Assert.NotEqual(maliciousPath, result); + } + + [WindowsOnlyFact] + public void ValidateLogFilePath_ShouldAcceptUserProfileTempPath() + { + // Use a fake server temp that differs from the user's profile temp, + // forcing the validation to exercise the profile-based lookup path. + string fakeServerTemp = @"C:\Windows\Temp"; + string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + string userTempPath = Path.Combine(userProfile, "AppData", "Local", "Temp", "Microsoft.NET.Workload_test.log"); + + string result = WindowsUtils.ValidateLogFilePath(userTempPath, fakeServerTemp); + Assert.Equal(Path.GetFullPath(userTempPath), result); + } + + [WindowsOnlyFact] + public void ValidateLogFilePath_ShouldRejectTraversalAttack() + { + string traversalPath = Path.Combine(Path.GetTempPath(), @"..\..\Windows\System32\evil.log"); + + string result = WindowsUtils.ValidateLogFilePath(traversalPath); + string canonicalized = Path.GetFullPath(traversalPath); + + // The traversal resolves to a system path, so it should be redirected + Assert.NotEqual(canonicalized, result); + Assert.StartsWith(Path.GetFullPath(Path.GetTempPath()), result, StringComparison.OrdinalIgnoreCase); + } + + [WindowsOnlyFact] + public void ValidatePackagePath_ShouldRejectTraversalAttack() + { + string cacheRoot = @"C:\ProgramData\dotnet\workloads"; + string traversalPath = cacheRoot + @"\..\..\..\..\Users\Public\evil.msi"; + + Assert.False(WindowsUtils.ValidatePackagePath(traversalPath, cacheRoot)); + } + + [WindowsOnlyFact] + public void ValidatePackagePath_ShouldRejectSiblingPrefixAttack() + { + string cacheRoot = @"C:\ProgramData\dotnet\workloads"; + string siblingPath = @"C:\ProgramData\dotnet\workloadsEvil\evil.msi"; + + Assert.False(WindowsUtils.ValidatePackagePath(siblingPath, cacheRoot)); + } + + [WindowsOnlyFact] + public void ValidatePackagePath_ShouldAcceptValidCachePath() + { + string cacheRoot = @"C:\ProgramData\dotnet\workloads"; + string validPath = @"C:\ProgramData\dotnet\workloads\pack\1.0\pack.msi"; + + Assert.True(WindowsUtils.ValidatePackagePath(validPath, cacheRoot)); + } + + [WindowsOnlyFact] + public void ValidatePathComponent_ShouldRejectTraversalSequence() + { + Assert.False(WindowsUtils.ValidatePathComponent(@"..\..\evil")); + } + + [WindowsOnlyFact] + public void ValidatePathComponent_ShouldRejectDirectorySeparator() + { + Assert.False(WindowsUtils.ValidatePathComponent(@"good\evil")); + } + + [WindowsOnlyFact] + public void ValidatePathComponent_ShouldRejectAltDirectorySeparator() + { + Assert.False(WindowsUtils.ValidatePathComponent("good/evil")); + } + + [WindowsOnlyFact] + public void ValidatePathComponent_ShouldRejectEmpty() + { + Assert.False(WindowsUtils.ValidatePathComponent("")); + Assert.False(WindowsUtils.ValidatePathComponent(null)); + } + + [WindowsOnlyFact] + public void ValidatePathComponent_ShouldAcceptValidComponent() + { + Assert.True(WindowsUtils.ValidatePathComponent("Microsoft.NET.Workload.Mono.ToolChain")); + Assert.True(WindowsUtils.ValidatePathComponent("8.0.100")); + } + + [WindowsOnlyFact] + public void ValidatePathUnderRoot_ShouldRejectTraversal() + { + string root = @"C:\ProgramData\dotnet\workloads"; + string traversal = root + @"\..\..\..\..\Windows\System32\evil.msi"; + + Assert.False(WindowsUtils.ValidatePathUnderRoot(traversal, root)); + } + + [WindowsOnlyFact] + public void ValidatePathUnderRoot_ShouldRejectSiblingPrefix() + { + string root = @"C:\ProgramData\dotnet\workloads"; + string sibling = @"C:\ProgramData\dotnet\workloadsEvil\evil.msi"; + + Assert.False(WindowsUtils.ValidatePathUnderRoot(sibling, root)); + } + + [WindowsOnlyFact] + public void ValidatePathUnderRoot_ShouldAcceptValidPath() + { + string root = @"C:\ProgramData\dotnet\workloads"; + string valid = @"C:\ProgramData\dotnet\workloads\pack\1.0\manifest.json"; + + Assert.True(WindowsUtils.ValidatePathUnderRoot(valid, root)); + } } [SupportedOSPlatform("windows")] From 831518bd3f929c28a777f5b9c501c44c88a2a582 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 6 May 2026 15:40:56 -0700 Subject: [PATCH 02/29] validate manifest path via passing --client-temp --client-temp must be passed because: 1. user process client may have a terminal with a different TEMP env var or TMP setting 2. sever may try to resolve TEMP to sever TEMP that does not grant reading to client (needed for manifest path) 3. user process under local only privlleges is still is a concern even if SIDs match after the point in time UAC and consolidating the --client-temp to be included within the UAC time window removes this concern 4. it is not possible to observe client temp from the server process with 100% authenticity 5. log temp resolution may fallback but falling back for the manifest path will cause a failure 6. the client and server should both always know about --client-temp because the same host is used to launch both 7. confirmed with workloads code author no further risks we could consider adding it as an optional arg to elevate --- .../Windows/InstallClientElevationContext.cs | 8 +- .../dotnet/Installer/Windows/InstallerBase.cs | 9 ++ .../Installer/Windows/MsiPackageCache.cs | 11 +- .../dotnet/Installer/Windows/WindowsUtils.cs | 87 +++++++++-- .../elevate/WorkloadElevateCommandParser.cs | 13 ++ test/dotnet.Tests/WindowsInstallerTests.cs | 135 +++++++++++++++++- 6 files changed, 245 insertions(+), 18 deletions(-) diff --git a/src/Cli/dotnet/Installer/Windows/InstallClientElevationContext.cs b/src/Cli/dotnet/Installer/Windows/InstallClientElevationContext.cs index 54dd62eaf0c9..dcc04f27a67e 100644 --- a/src/Cli/dotnet/Installer/Windows/InstallClientElevationContext.cs +++ b/src/Cli/dotnet/Installer/Windows/InstallClientElevationContext.cs @@ -32,10 +32,16 @@ public override void Elevate() { if (!IsElevated && !HasElevated) { + // Pass the unelevated client's temp directory to the elevated server so it can validate + // IPC-supplied paths (e.g., the workload pack manifest extracted by the client) against it. + // Quoted to handle profile paths that contain spaces. Optional on the server side; if + // omitted or unparseable, the server falls back to its own Path.GetTempPath(). + string clientTemp = Path.GetFullPath(Path.GetTempPath()).TrimEnd(Path.DirectorySeparatorChar); + // Use the path of the current host, otherwise we risk resolving against the wrong SDK version. // To trigger UAC, UseShellExecute must be true and Verb must be "runas". ProcessStartInfo startInfo = new($@"""{Environment.ProcessPath}""", - $@"""{Assembly.GetExecutingAssembly().Location}"" workload elevate") + $@"""{Assembly.GetExecutingAssembly().Location}"" workload elevate --client-temp ""{clientTemp}""") { Verb = "runas", UseShellExecute = true, diff --git a/src/Cli/dotnet/Installer/Windows/InstallerBase.cs b/src/Cli/dotnet/Installer/Windows/InstallerBase.cs index 6ac423a25ea6..b631f4511a28 100644 --- a/src/Cli/dotnet/Installer/Windows/InstallerBase.cs +++ b/src/Cli/dotnet/Installer/Windows/InstallerBase.cs @@ -66,6 +66,15 @@ protected InstallElevationContextBase ElevationContext /// public static readonly Process ParentProcess; + /// + /// The fully-qualified path of the unelevated client's temp directory, as supplied at server launch + /// via the --client-temp argument. Used by path validators to accept manifest/log paths that + /// originate from the client when the client and server resolve different values for + /// (e.g., over-the-shoulder UAC, custom TEMP env vars). + /// if not supplied. + /// + public static string TrustedClientTempDirectory { get; set; } + /// /// Gets the processor architecture. /// diff --git a/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs b/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs index 98359b398fe9..ecda79993c7b 100644 --- a/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs +++ b/src/Cli/dotnet/Installer/Windows/MsiPackageCache.cs @@ -55,10 +55,17 @@ public void CachePayload(string packageId, string packageVersion, string manifes throw new ArgumentException($"Invalid package version: {packageVersion}"); } - // Validate that the manifest path resolves to a location under the package cache root - // or the user's temp directory (where packages are extracted before caching). + // Validate that the manifest path resolves to a location under the elevated server's temp + // directory or the unelevated client's temp directory (when supplied at server launch via + // --client-temp). This prevents an IPC client from coercing the elevated server into reading + // or moving arbitrary files. string fullManifestPath = Path.GetFullPath(manifestPath); + if (!WindowsUtils.ValidateManifestPath(fullManifestPath)) + { + throw new ArgumentException($"CachePayload: Manifest path is not under an allowed temp directory: {manifestPath}"); + } + if (!File.Exists(fullManifestPath)) { throw new FileNotFoundException($"CachePayload: Manifest file not found: {fullManifestPath}"); diff --git a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs index 2f38f7c11f31..1b2df95becf9 100644 --- a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs +++ b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs @@ -15,7 +15,7 @@ using Microsoft.Win32; using Microsoft.Win32.SafeHandles; -namespace Microsoft.DotNet.Cli.Installer.Windows; +namespace Microsoft.DotNet.Installer.Windows; [SupportedOSPlatform("windows")] /// @@ -134,7 +134,8 @@ public static PipeSecurity CreatePipeSecurity(SecurityIdentifier ownerSid, Secur /// /// Validates and returns the log file path to use for MSI operations. - /// Ensures the path is under the server's temp directory or the parent user's profile temp directory. + /// Ensures the path is under the server's temp directory, the trusted client temp directory + /// (if supplied at server launch), or the parent user's profile temp directory. /// If the path is not in an allowed location, it is redirected to the server's temp directory. /// /// The requested log file path. @@ -148,7 +149,16 @@ public static string ValidateLogFilePath(string logFile, string serverTempPath = string serverTemp = Path.GetFullPath(serverTempPath ?? Path.GetTempPath()); // Fast path: log file is under the server's own temp directory. - if (fullLogPath.StartsWith(serverTemp, StringComparison.OrdinalIgnoreCase)) + if (IsPathUnder(fullLogPath, serverTemp)) + { + return fullLogPath; + } + + // Trusted client-supplied temp: the unelevated client tells the server its Path.GetTempPath() + // value at launch via the --client-temp argument. Honors custom TEMP env vars and over-the-shoulder + // UAC scenarios where the elevated server resolves a different temp directory than the client. + string clientTemp = InstallerBase.TrustedClientTempDirectory; + if (!string.IsNullOrEmpty(clientTemp) && IsPathUnder(fullLogPath, clientTemp)) { return fullLogPath; } @@ -166,8 +176,7 @@ public static string ValidateLogFilePath(string logFile, string serverTempPath = if (profilePath != null) { string profileTemp = Path.GetFullPath(Path.Combine(profilePath, "AppData", "Local", "Temp")); - if (fullLogPath.StartsWith(profileTemp + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) - || fullLogPath.Equals(profileTemp, StringComparison.OrdinalIgnoreCase)) + if (IsPathUnder(fullLogPath, profileTemp)) { return fullLogPath; } @@ -183,6 +192,61 @@ public static string ValidateLogFilePath(string logFile, string serverTempPath = return Path.Combine(serverTemp, Path.GetFileName(fullLogPath)); } + /// + /// Validates that an IPC-supplied workload manifest path lives under an allowed root, to prevent + /// a same-user attacker on the install pipe from coercing the elevated server into reading or moving + /// arbitrary files. Allowed roots are the server's own temp directory and the trusted client temp + /// directory (if supplied at server launch via --client-temp). + /// + /// The manifest path supplied by the client. + /// The server's temp directory. If null, defaults to . + /// if the canonicalized path is under an allowed root; otherwise . + public static bool ValidateManifestPath(string manifestPath, string serverTempPath = null) + { + if (string.IsNullOrWhiteSpace(manifestPath)) + { + return false; + } + + string fullManifestPath; + try + { + fullManifestPath = Path.GetFullPath(manifestPath); + } + catch + { + return false; + } + + string serverTemp = Path.GetFullPath(serverTempPath ?? Path.GetTempPath()); + if (IsPathUnder(fullManifestPath, serverTemp)) + { + return true; + } + + string clientTemp = InstallerBase.TrustedClientTempDirectory; + if (!string.IsNullOrEmpty(clientTemp) && IsPathUnder(fullManifestPath, clientTemp)) + { + return true; + } + + return false; + } + + /// + /// Returns true if equals or is contained beneath . + /// Both inputs must already be canonicalized via . + /// + private static bool IsPathUnder(string fullPath, string root) + { + string rootWithSep = root.EndsWith(Path.DirectorySeparatorChar) + ? root + : root + Path.DirectorySeparatorChar; + + return fullPath.Equals(root, StringComparison.OrdinalIgnoreCase) + || fullPath.StartsWith(rootWithSep, StringComparison.OrdinalIgnoreCase); + } + /// /// Returns the profile path for the user identified by the specified . /// Reads the ProfileImagePath value from the registry ProfileList key. @@ -230,21 +294,18 @@ public static bool ValidatePathComponent(string component) } /// - /// Validates that the specified path, after canonicalization, is under the expected root directory. - /// Prevents directory traversal and sibling-prefix attacks. + /// Validates that the specified path, after canonicalization, is the expected root directory itself + /// or is contained beneath it. Prevents directory traversal and sibling-prefix attacks + /// (e.g., C:\Temp2\evil against root C:\Temp). /// /// The path to validate. /// The expected root directory. - /// if the canonicalized path is under the root; otherwise . + /// if the canonicalized path equals or is under the root; otherwise . public static bool ValidatePathUnderRoot(string path, string expectedRoot) { string fullPath = Path.GetFullPath(path); string fullRoot = Path.GetFullPath(expectedRoot); - if (!fullRoot.EndsWith(Path.DirectorySeparatorChar)) - { - fullRoot += Path.DirectorySeparatorChar; - } - return fullPath.StartsWith(fullRoot, StringComparison.OrdinalIgnoreCase); + return IsPathUnder(fullPath, fullRoot); } } diff --git a/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommandParser.cs b/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommandParser.cs index acfaf3c71e13..e84095ff56d3 100644 --- a/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommandParser.cs +++ b/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommandParser.cs @@ -9,6 +9,17 @@ namespace Microsoft.DotNet.Cli { internal static class WorkloadElevateCommandParser { + /// + /// Optional, hidden argument supplied by the unelevated client at server launch with the value of + /// the client's . Used by the elevated server to accept + /// IPC-supplied paths that originate from the client's temp directory when it differs from the + /// server's (e.g., over-the-shoulder UAC, custom TEMP env vars). + /// + public static readonly CliOption ClientTempOption = new("--client-temp") + { + Hidden = true + }; + private static readonly CliCommand Command = ConstructCommand(); public static CliCommand GetCommand() @@ -23,6 +34,8 @@ private static CliCommand ConstructCommand() Hidden = true }; + command.Options.Add(ClientTempOption); + command.SetAction((parseResult) => new WorkloadElevateCommand(parseResult).Execute()); return command; diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index 78409baa9e19..10fb945cb884 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -8,8 +8,8 @@ using System.Runtime.Versioning; using System.Security.AccessControl; using System.Security.Principal; -using Microsoft.DotNet.Cli.Installer.Windows; -using Microsoft.DotNet.Cli.Installer.Windows.Security; +using Microsoft.DotNet.Installer.Windows; +using Microsoft.DotNet.Installer.Windows.Security; namespace Microsoft.DotNet.Tests { @@ -324,6 +324,137 @@ public void ValidatePathUnderRoot_ShouldAcceptValidPath() Assert.True(WindowsUtils.ValidatePathUnderRoot(valid, root)); } + + [WindowsOnlyFact] + public void ValidatePathUnderRoot_ShouldAcceptRootItself() + { + string root = @"C:\ProgramData\dotnet\workloads"; + + Assert.True(WindowsUtils.ValidatePathUnderRoot(root, root)); + Assert.True(WindowsUtils.ValidatePathUnderRoot(root + @"\", root)); + Assert.True(WindowsUtils.ValidatePathUnderRoot(root, root + @"\")); + } + + [WindowsOnlyFact] + public void ValidateManifestPath_ShouldAcceptPathUnderServerTemp() + { + string serverTemp = Path.GetFullPath(Path.GetTempPath()).TrimEnd(Path.DirectorySeparatorChar); + string manifest = Path.Combine(serverTemp, Guid.NewGuid().ToString(), "data", "msi.json"); + + string priorClientTemp = InstallerBase.TrustedClientTempDirectory; + try + { + InstallerBase.TrustedClientTempDirectory = null; + Assert.True(WindowsUtils.ValidateManifestPath(manifest)); + } + finally + { + InstallerBase.TrustedClientTempDirectory = priorClientTemp; + } + } + + [WindowsOnlyFact] + public void ValidateManifestPath_ShouldAcceptPathUnderTrustedClientTemp() + { + string fakeServerTemp = @"C:\fake-server-temp"; + string fakeClientTemp = @"C:\fake-client-temp"; + string manifest = Path.Combine(fakeClientTemp, Guid.NewGuid().ToString(), "data", "msi.json"); + + string priorClientTemp = InstallerBase.TrustedClientTempDirectory; + try + { + InstallerBase.TrustedClientTempDirectory = fakeClientTemp; + Assert.True(WindowsUtils.ValidateManifestPath(manifest, fakeServerTemp)); + } + finally + { + InstallerBase.TrustedClientTempDirectory = priorClientTemp; + } + } + + [WindowsOnlyFact] + public void ValidateManifestPath_ShouldRejectPathOutsideAllowedRoots() + { + string fakeServerTemp = @"C:\fake-server-temp"; + string maliciousPath = @"C:\Users\OtherUser\Desktop\evil.json"; + + string priorClientTemp = InstallerBase.TrustedClientTempDirectory; + try + { + InstallerBase.TrustedClientTempDirectory = null; + Assert.False(WindowsUtils.ValidateManifestPath(maliciousPath, fakeServerTemp)); + } + finally + { + InstallerBase.TrustedClientTempDirectory = priorClientTemp; + } + } + + [WindowsOnlyFact] + public void ValidateManifestPath_ShouldRejectTraversalAttack() + { + string serverTemp = Path.GetFullPath(Path.GetTempPath()).TrimEnd(Path.DirectorySeparatorChar); + string traversal = Path.Combine(serverTemp, "..", "..", "..", "Windows", "System32", "evil.json"); + + string priorClientTemp = InstallerBase.TrustedClientTempDirectory; + try + { + InstallerBase.TrustedClientTempDirectory = null; + Assert.False(WindowsUtils.ValidateManifestPath(traversal)); + } + finally + { + InstallerBase.TrustedClientTempDirectory = priorClientTemp; + } + } + + [WindowsOnlyFact] + public void ValidateManifestPath_ShouldRejectSiblingPrefix() + { + // Ensure that a path that shares a prefix with an allowed root but lies outside it is rejected + // (e.g., C:\Temp vs C:\Temp_evil). + string fakeServerTemp = @"C:\fake-server-temp"; + string sibling = @"C:\fake-server-temp_evil\msi.json"; + + string priorClientTemp = InstallerBase.TrustedClientTempDirectory; + try + { + InstallerBase.TrustedClientTempDirectory = null; + Assert.False(WindowsUtils.ValidateManifestPath(sibling, fakeServerTemp)); + } + finally + { + InstallerBase.TrustedClientTempDirectory = priorClientTemp; + } + } + + [WindowsOnlyFact] + public void ValidateManifestPath_ShouldRejectNullOrEmpty() + { + Assert.False(WindowsUtils.ValidateManifestPath(null)); + Assert.False(WindowsUtils.ValidateManifestPath("")); + Assert.False(WindowsUtils.ValidateManifestPath(" ")); + } + + [WindowsOnlyFact] + public void ValidateLogFilePath_ShouldAcceptTrustedClientTemp() + { + string fakeServerTemp = @"C:\fake-server-temp"; + string fakeClientTemp = @"C:\fake-client-temp"; + string clientLogPath = Path.Combine(fakeClientTemp, "Microsoft.NET.Workload_42.log"); + + string priorClientTemp = InstallerBase.TrustedClientTempDirectory; + try + { + InstallerBase.TrustedClientTempDirectory = fakeClientTemp; + string result = WindowsUtils.ValidateLogFilePath(clientLogPath, fakeServerTemp); + Assert.Equal(Path.GetFullPath(clientLogPath), result); + } + finally + { + InstallerBase.TrustedClientTempDirectory = priorClientTemp; + } + } } [SupportedOSPlatform("windows")] From fb87e78732053517b22ee6cac48399dde29f26bc Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 6 May 2026 15:43:20 -0700 Subject: [PATCH 03/29] canonicalize client temp arg in elevation --- .../elevate/WorkloadElevateCommand.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs b/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs index c64f2e656baf..4e0c23d13be2 100644 --- a/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs +++ b/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.CommandLine; +using Microsoft.DotNet.Cli; using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.Installer.Windows; using Microsoft.DotNet.Workloads.Workload.Install; namespace Microsoft.DotNet.Workloads.Workload.Elevate @@ -21,6 +23,22 @@ public override int Execute() { try { + // Capture the unelevated client's temp directory (if supplied) so path validators + // can accept IPC-supplied paths that originate from it. Optional and ignored when null + // or unparseable; in either case validators fall back to the server's own temp. + string clientTemp = _parseResult.GetValue(WorkloadElevateCommandParser.ClientTempOption); + if (!string.IsNullOrWhiteSpace(clientTemp)) + { + try + { + InstallerBase.TrustedClientTempDirectory = System.IO.Path.GetFullPath(clientTemp); + } + catch + { + // Ignore malformed values. + } + } + _server = NetSdkMsiInstallerServer.Create(VerifySignatures); _server.Run(); } From 57aee5f6e08fb80ad37ef3b68c6e827a4beadbd5 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 6 May 2026 16:36:39 -0700 Subject: [PATCH 04/29] code cleanup --- src/Cli/dotnet/Installer/Windows/WindowsUtils.cs | 2 +- test/dotnet.Tests/WindowsInstallerTests.cs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs index 1b2df95becf9..e27bb3a28b0f 100644 --- a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs +++ b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs @@ -125,7 +125,7 @@ public static PipeSecurity CreatePipeSecurity(SecurityIdentifier ownerSid, Secur pipeSecurity.SetOwner(ownerSid); pipeSecurity.AddAccessRule(new PipeAccessRule(ownerSid, PipeAccessRights.FullControl, AccessControlType.Allow)); - // Restrict read/write access to authenticated users + // Restrict read/write access to the allowed client (typically in workloads the unelevated process parent talking to the elevated 'server') pipeSecurity.AddAccessRule(new PipeAccessRule(clientSid, PipeAccessRights.Read | PipeAccessRights.Write | PipeAccessRights.Synchronize, AccessControlType.Allow)); diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index 10fb945cb884..968c85e1b5cd 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -239,6 +239,18 @@ public void ValidateLogFilePath_ShouldRejectTraversalAttack() Assert.StartsWith(Path.GetFullPath(Path.GetTempPath()), result, StringComparison.OrdinalIgnoreCase); } + [WindowsOnlyFact] + public void ValidateLogFilePath_ShouldRejectSiblingPrefixAttack() + { + // "C:\TempEvil" should NOT be accepted when serverTemp is "C:\Temp" + string serverTemp = @"C:\Temp"; + string maliciousPath = @"C:\TempEvil\evil.log"; + + string result = WindowsUtils.ValidateLogFilePath(maliciousPath, serverTemp); + Assert.NotEqual(Path.GetFullPath(maliciousPath), result); + Assert.StartsWith(Path.GetFullPath(serverTemp), result, StringComparison.OrdinalIgnoreCase); + } + [WindowsOnlyFact] public void ValidatePackagePath_ShouldRejectTraversalAttack() { From f1c98e06ece1795eddf009c80b6af2eadcc27445 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 6 May 2026 16:53:17 -0700 Subject: [PATCH 05/29] fix path bug --- .../dotnet/Installer/Windows/WindowsUtils.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs index e27bb3a28b0f..70cb894b2be1 100644 --- a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs +++ b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs @@ -239,12 +239,14 @@ public static bool ValidateManifestPath(string manifestPath, string serverTempPa /// private static bool IsPathUnder(string fullPath, string root) { - string rootWithSep = root.EndsWith(Path.DirectorySeparatorChar) - ? root - : root + Path.DirectorySeparatorChar; - - return fullPath.Equals(root, StringComparison.OrdinalIgnoreCase) - || fullPath.StartsWith(rootWithSep, StringComparison.OrdinalIgnoreCase); + // Normalize both paths by trimming trailing separators so that + // "C:\Temp\" and "C:\Temp" compare identically. + string normalizedPath = fullPath.TrimEnd(Path.DirectorySeparatorChar); + string normalizedRoot = root.TrimEnd(Path.DirectorySeparatorChar); + string rootWithSep = normalizedRoot + Path.DirectorySeparatorChar; + + return normalizedPath.Equals(normalizedRoot, StringComparison.OrdinalIgnoreCase) + || normalizedPath.StartsWith(rootWithSep, StringComparison.OrdinalIgnoreCase); } /// @@ -257,10 +259,13 @@ public static string GetUserProfilePath(SecurityIdentifier sid) { // HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList\{SID}\ProfileImagePath // contains the full path to the user's profile directory (e.g., C:\Users\username). + // RegistryKey.GetValue expands REG_EXPAND_SZ values by default, but call ExpandEnvironmentVariables + // explicitly to also handle the rare case where the value was stored as REG_SZ with literal %vars%. using RegistryKey profileListKey = Registry.LocalMachine.OpenSubKey( $@"SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList\{sid.Value}"); - return profileListKey?.GetValue("ProfileImagePath") as string; + string profileImagePath = profileListKey?.GetValue("ProfileImagePath") as string; + return profileImagePath != null ? Environment.ExpandEnvironmentVariables(profileImagePath) : null; } /// From ea73392615145ab5d8bb49e26902f794c81befd2 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 7 May 2026 10:03:21 -0700 Subject: [PATCH 06/29] collapse ipc validation tests into theories --- .../elevate/WorkloadElevateCommand.cs | 3 +- test/dotnet.Tests/WindowsInstallerTests.cs | 84 +++++-------------- 2 files changed, 25 insertions(+), 62 deletions(-) diff --git a/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs b/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs index 4e0c23d13be2..be85e8b6eb91 100644 --- a/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs +++ b/src/Cli/dotnet/commands/dotnet-workload/elevate/WorkloadElevateCommand.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.CommandLine; +using System.IO; using Microsoft.DotNet.Cli; using Microsoft.DotNet.Cli.Utils; using Microsoft.DotNet.Installer.Windows; @@ -31,7 +32,7 @@ public override int Execute() { try { - InstallerBase.TrustedClientTempDirectory = System.IO.Path.GetFullPath(clientTemp); + InstallerBase.TrustedClientTempDirectory = Path.GetFullPath(clientTemp); } catch { diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index 968c85e1b5cd..bdccacc06750 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -278,73 +278,35 @@ public void ValidatePackagePath_ShouldAcceptValidCachePath() Assert.True(WindowsUtils.ValidatePackagePath(validPath, cacheRoot)); } - [WindowsOnlyFact] - public void ValidatePathComponent_ShouldRejectTraversalSequence() - { - Assert.False(WindowsUtils.ValidatePathComponent(@"..\..\evil")); - } - - [WindowsOnlyFact] - public void ValidatePathComponent_ShouldRejectDirectorySeparator() - { - Assert.False(WindowsUtils.ValidatePathComponent(@"good\evil")); - } - - [WindowsOnlyFact] - public void ValidatePathComponent_ShouldRejectAltDirectorySeparator() - { - Assert.False(WindowsUtils.ValidatePathComponent("good/evil")); - } - - [WindowsOnlyFact] - public void ValidatePathComponent_ShouldRejectEmpty() - { - Assert.False(WindowsUtils.ValidatePathComponent("")); - Assert.False(WindowsUtils.ValidatePathComponent(null)); - } - - [WindowsOnlyFact] - public void ValidatePathComponent_ShouldAcceptValidComponent() - { - Assert.True(WindowsUtils.ValidatePathComponent("Microsoft.NET.Workload.Mono.ToolChain")); - Assert.True(WindowsUtils.ValidatePathComponent("8.0.100")); - } - - [WindowsOnlyFact] - public void ValidatePathUnderRoot_ShouldRejectTraversal() + [WindowsOnlyTheory] + [InlineData(@"..\..\evil")] + [InlineData(@"good\evil")] + [InlineData("good/evil")] + [InlineData("")] + [InlineData(null)] + public void ValidatePathComponent_ShouldRejectInvalidInput(string input) { - string root = @"C:\ProgramData\dotnet\workloads"; - string traversal = root + @"\..\..\..\..\Windows\System32\evil.msi"; - - Assert.False(WindowsUtils.ValidatePathUnderRoot(traversal, root)); + Assert.False(WindowsUtils.ValidatePathComponent(input)); } - [WindowsOnlyFact] - public void ValidatePathUnderRoot_ShouldRejectSiblingPrefix() - { - string root = @"C:\ProgramData\dotnet\workloads"; - string sibling = @"C:\ProgramData\dotnet\workloadsEvil\evil.msi"; - - Assert.False(WindowsUtils.ValidatePathUnderRoot(sibling, root)); - } - - [WindowsOnlyFact] - public void ValidatePathUnderRoot_ShouldAcceptValidPath() + [WindowsOnlyTheory] + [InlineData("Microsoft.NET.Workload.Mono.ToolChain")] + [InlineData("8.0.100")] + public void ValidatePathComponent_ShouldAcceptValidComponent(string input) { - string root = @"C:\ProgramData\dotnet\workloads"; - string valid = @"C:\ProgramData\dotnet\workloads\pack\1.0\manifest.json"; - - Assert.True(WindowsUtils.ValidatePathUnderRoot(valid, root)); + Assert.True(WindowsUtils.ValidatePathComponent(input)); } - [WindowsOnlyFact] - public void ValidatePathUnderRoot_ShouldAcceptRootItself() - { - string root = @"C:\ProgramData\dotnet\workloads"; - - Assert.True(WindowsUtils.ValidatePathUnderRoot(root, root)); - Assert.True(WindowsUtils.ValidatePathUnderRoot(root + @"\", root)); - Assert.True(WindowsUtils.ValidatePathUnderRoot(root, root + @"\")); + [WindowsOnlyTheory] + [InlineData(@"C:\ProgramData\dotnet\workloads\..\..\..\..\Windows\System32\evil.msi", @"C:\ProgramData\dotnet\workloads", false)] + [InlineData(@"C:\ProgramData\dotnet\workloadsEvil\evil.msi", @"C:\ProgramData\dotnet\workloads", false)] + [InlineData(@"C:\ProgramData\dotnet\workloads\pack\1.0\manifest.json", @"C:\ProgramData\dotnet\workloads", true)] + [InlineData(@"C:\ProgramData\dotnet\workloads", @"C:\ProgramData\dotnet\workloads", true)] + [InlineData(@"C:\ProgramData\dotnet\workloads\", @"C:\ProgramData\dotnet\workloads", true)] + [InlineData(@"C:\ProgramData\dotnet\workloads", @"C:\ProgramData\dotnet\workloads\", true)] + public void ValidatePathUnderRoot_ReturnsExpectedResult(string path, string root, bool expected) + { + Assert.Equal(expected, WindowsUtils.ValidatePathUnderRoot(path, root)); } [WindowsOnlyFact] From ec5505115938efe2a65c2e062e1be3fcb0264a5b Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 7 May 2026 10:05:04 -0700 Subject: [PATCH 07/29] fix build error removing bad import --- src/Cli/dotnet/Installer/Windows/WindowsUtils.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs index 70cb894b2be1..67427f64e6dc 100644 --- a/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs +++ b/src/Cli/dotnet/Installer/Windows/WindowsUtils.cs @@ -11,7 +11,6 @@ using System.Security.Principal; using Microsoft.DotNet.Cli.Telemetry; using Microsoft.DotNet.Cli.Utils; -using Microsoft.DotNet.Utilities; using Microsoft.Win32; using Microsoft.Win32.SafeHandles; From 144695724f6660db51723fb5df7581a5d86128c1 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 7 May 2026 10:45:50 -0700 Subject: [PATCH 08/29] fix nullable directive in windows test --- test/dotnet.Tests/WindowsInstallerTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index bdccacc06750..7b1a51ef1170 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable disable - using System.IO.Pipes; using System.Reflection; using System.Runtime.Versioning; From 9ff5989d008f3a7f60a3340c4bc41ac628c3dab5 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 7 May 2026 16:31:09 -0700 Subject: [PATCH 09/29] fix installer test json - json format expectation changed --- test/dotnet.Tests/WindowsInstallerTests.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index 7b1a51ef1170..1b748ab23855 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -80,19 +80,21 @@ public void InstallMessageDispatcherProcessesMessages() } [WindowsOnlyFact] - public void InstallRequestMessageCreateThrowsForNullPayload() + public void InstallRequestMessageCreateReturnsDefaultForNullPayload() { - Action action = () => InstallRequestMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); + InstallRequestMessage message = InstallRequestMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); - action.Should().Throw(); + message.Should().NotBeNull(); + message.RequestType.Should().Be(default); } [WindowsOnlyFact] - public void InstallResponseMessageCreateThrowsForNullPayload() + public void InstallResponseMessageCreateReturnsDefaultForNullPayload() { - Action action = () => InstallResponseMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); + InstallResponseMessage message = InstallResponseMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); - action.Should().Throw(); + message.Should().NotBeNull(); + message.Message.Should().BeNull(); } [WindowsOnlyTheory] From 812946d9cd29f9bf914b6c850f1392d76755b528 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Fri, 8 May 2026 11:07:18 -0700 Subject: [PATCH 10/29] fix busted json test --- test/dotnet.Tests/WindowsInstallerTests.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/dotnet.Tests/WindowsInstallerTests.cs b/test/dotnet.Tests/WindowsInstallerTests.cs index 1b748ab23855..d8e8f509cdbe 100644 --- a/test/dotnet.Tests/WindowsInstallerTests.cs +++ b/test/dotnet.Tests/WindowsInstallerTests.cs @@ -79,24 +79,6 @@ public void InstallMessageDispatcherProcessesMessages() Assert.Equal("Shutting down!", r2.Message); } - [WindowsOnlyFact] - public void InstallRequestMessageCreateReturnsDefaultForNullPayload() - { - InstallRequestMessage message = InstallRequestMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); - - message.Should().NotBeNull(); - message.RequestType.Should().Be(default); - } - - [WindowsOnlyFact] - public void InstallResponseMessageCreateReturnsDefaultForNullPayload() - { - InstallResponseMessage message = InstallResponseMessage.Create(System.Text.Encoding.UTF8.GetBytes("null")); - - message.Should().NotBeNull(); - message.Message.Should().BeNull(); - } - [WindowsOnlyTheory] [InlineData("1033,1041,1049", UpgradeAttributes.MigrateFeatures, 1041, false)] [InlineData(null, UpgradeAttributes.LanguagesExclusive, 3082, false)] From 6dde6e5a07def4626776ce6c00ab256a112b1602 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 16 May 2026 02:01:55 +0000 Subject: [PATCH 11/29] Update dependencies from https://github.com/dotnet/source-build-externals build 20260515.1 On relative base path root Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.26216.1 -> To Version 8.0.0-alpha.1.26265.1 --- NuGet.config | 7 ------- eng/Version.Details.xml | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/NuGet.config b/NuGet.config index 2afd4326cfb1..2dd596731770 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,26 +4,22 @@ - - - - @@ -50,17 +46,14 @@ - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ecc32dbc4505..4cc343bef588 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -349,9 +349,9 @@ 02fe27cd6a9b001c8feb7938e6ef4b3799745759 - + https://github.com/dotnet/source-build-externals - 7abd7043b258fe69233070da30f6a88438f4b293 + 068e37394c61ba439d2898151c6c078dad3ef66b From 4bb6b0ea1875fe12e6662f67b4adfe85a6cff62f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 18 May 2026 02:55:54 +0000 Subject: [PATCH 12/29] Update dependencies from https://github.com/dotnet/templating build 20260517.1 On relative base path root Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Mocks From Version 8.0.128-servicing.26262.3 -> To Version 8.0.128-servicing.26267.1 Microsoft.TemplateEngine.Abstractions From Version 8.0.128 -> To Version 8.0.128 --- NuGet.config | 9 +-------- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/NuGet.config b/NuGet.config index 2663b23dc763..a0ac8e2f7208 100644 --- a/NuGet.config +++ b/NuGet.config @@ -3,13 +3,11 @@ - - @@ -17,13 +15,11 @@ - - + - @@ -48,7 +44,6 @@ - @@ -56,10 +51,8 @@ - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2c116ad38fdc..05c9b3e4f0f9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,15 +3,15 @@ https://github.com/dotnet/templating - 606b9e0ec41b75c994c5b942c13d77efb943fa2a + 16f951772c2dd2b72b22982f742b0deb88622870 - + https://github.com/dotnet/templating - 606b9e0ec41b75c994c5b942c13d77efb943fa2a + 16f951772c2dd2b72b22982f742b0deb88622870 - + https://github.com/dotnet/templating - 606b9e0ec41b75c994c5b942c13d77efb943fa2a + 16f951772c2dd2b72b22982f742b0deb88622870 diff --git a/eng/Versions.props b/eng/Versions.props index e1e4511c3802..abe1718993c8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -148,7 +148,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 8.0.128-servicing.26262.3 + 8.0.128-servicing.26267.1 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 374029dfa96139124b9b4b38040d5b1cc04d246c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 04:37:29 +0000 Subject: [PATCH 13/29] [release/8.0.1xx] Update dependencies from dotnet/source-build-externals (#54327) [release/8.0.1xx] Update dependencies from dotnet/source-build-externals --- NuGet.config | 7 ------- eng/Version.Details.xml | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/NuGet.config b/NuGet.config index 2663b23dc763..26a798df2e2d 100644 --- a/NuGet.config +++ b/NuGet.config @@ -3,13 +3,11 @@ - - @@ -17,13 +15,11 @@ - - @@ -48,7 +44,6 @@ - @@ -56,10 +51,8 @@ - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2c116ad38fdc..c50f9656af7c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -349,9 +349,9 @@ 02fe27cd6a9b001c8feb7938e6ef4b3799745759 - + https://github.com/dotnet/source-build-externals - 7abd7043b258fe69233070da30f6a88438f4b293 + 068e37394c61ba439d2898151c6c078dad3ef66b From 2c90fc83306e3e20d07a95cb10fd0e8f8b67fe04 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 04:38:54 +0000 Subject: [PATCH 14/29] Reset files to release/8.0.4xx Reset patterns: - global.json - NuGet.config - eng/Version.Details.xml - eng/Versions.props - eng/common/* --- NuGet.config | 13 +-- eng/Version.Details.xml | 186 ++++++++++++++++++++-------------------- eng/Versions.props | 95 ++++++++++---------- 3 files changed, 149 insertions(+), 145 deletions(-) diff --git a/NuGet.config b/NuGet.config index a0ac8e2f7208..2dd596731770 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,3 +1,4 @@ + @@ -5,22 +6,23 @@ - + - - + + + @@ -45,14 +47,15 @@ - + - + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 998ff0d4a1a4..4cc343bef588 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,17 +1,17 @@ - + https://github.com/dotnet/templating - 16f951772c2dd2b72b22982f742b0deb88622870 + 3dba2c56d483d784cf7241e2fef06b7f6c845d96 - + https://github.com/dotnet/templating - 16f951772c2dd2b72b22982f742b0deb88622870 + 3dba2c56d483d784cf7241e2fef06b7f6c845d96 - + https://github.com/dotnet/templating - 16f951772c2dd2b72b22982f742b0deb88622870 + 3dba2c56d483d784cf7241e2fef06b7f6c845d96 @@ -55,61 +55,61 @@ https://github.com/dotnet/emsdk 925a4f472f7e1c6d3bcb5092d28d952f90cc6237 - - https://github.com/dotnet/msbuild - 7806cbf7b0fd91ea6ab55c2e42d8ed973114e197 + + https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + 02bf66295b64ab368d12933041f7281aad186a2d - - https://github.com/dotnet/msbuild - 7806cbf7b0fd91ea6ab55c2e42d8ed973114e197 + + https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + 02bf66295b64ab368d12933041f7281aad186a2d - - https://github.com/dotnet/msbuild - 7806cbf7b0fd91ea6ab55c2e42d8ed973114e197 + + https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + 02bf66295b64ab368d12933041f7281aad186a2d - + https://github.com/dotnet/fsharp - fc5e9eda234e2b69aa479f4f83faddc31fdd4da7 + e11d7079bebc6f101c5313fe0d1de9e3d38a7c02 - + https://github.com/dotnet/fsharp - fc5e9eda234e2b69aa479f4f83faddc31fdd4da7 + e11d7079bebc6f101c5313fe0d1de9e3d38a7c02 - + https://github.com/dotnet/format - a8be5ff9c558f921f565c461dd7688905f84742a + b7d483d5a0ce4e96a3468514f3f4a98f3c371434 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 - + https://github.com/dotnet/roslyn - 38896ab4e7cee896fcde8a4e26914a777c794e3b + 3fb752d448006a3144a60ccf181d745e555422f9 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -119,86 +119,86 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore be2530c3035e4bfa7670c6b18f5a64ef89e0e80d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 60584b9c2451c0e324655f3968bc4ed1a1a8c85c + c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 - + https://github.com/microsoft/vstest - aa59400b11e1aeee2e8af48928dbd48748a8bef9 + 7855c9b221686104532ebf3380f2d45b3613b369 - + https://github.com/microsoft/vstest - aa59400b11e1aeee2e8af48928dbd48748a8bef9 + 7855c9b221686104532ebf3380f2d45b3613b369 - + https://github.com/microsoft/vstest - aa59400b11e1aeee2e8af48928dbd48748a8bef9 + 7855c9b221686104532ebf3380f2d45b3613b369 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -293,18 +293,18 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore be2530c3035e4bfa7670c6b18f5a64ef89e0e80d - + https://github.com/dotnet/razor - e137c0322ddb9e3d4f4a5de385647baa9719f9a6 + 2f31e7f15e35ce060c1ba81174a87fdd0ae9f479 - + https://github.com/dotnet/razor - e137c0322ddb9e3d4f4a5de385647baa9719f9a6 + 2f31e7f15e35ce060c1ba81174a87fdd0ae9f479 - + https://github.com/dotnet/razor - e137c0322ddb9e3d4f4a5de385647baa9719f9a6 + 2f31e7f15e35ce060c1ba81174a87fdd0ae9f479 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -327,17 +327,17 @@ 9a1c3e1b7f0c8763d4c96e593961a61a72679a7b - + https://github.com/dotnet/roslyn-analyzers - 0963af6bfc2a3d5083faa2b7c1c91a4eee29a6f4 + abef8ced132657943b7150f01a308e2199a17d5d - + https://github.com/dotnet/roslyn-analyzers - 0963af6bfc2a3d5083faa2b7c1c91a4eee29a6f4 + abef8ced132657943b7150f01a308e2199a17d5d - + https://github.com/dotnet/roslyn-analyzers - 0963af6bfc2a3d5083faa2b7c1c91a4eee29a6f4 + abef8ced132657943b7150f01a308e2199a17d5d @@ -354,10 +354,10 @@ 068e37394c61ba439d2898151c6c078dad3ef66b - - https://github.com/dotnet/source-build-assets - 43fa27099796cf60357226180f279946dbb6495e - + + https://github.com/dotnet/source-build-reference-packages + 9d83c59ff890ff2020aa38ebcf3bb514e38e9ffe + https://github.com/dotnet/deployment-tools @@ -469,9 +469,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 81cabf2857a01351e5ab578947c7403a5b128ad1 - + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - e4ede9b8979b9d2b1b1d4383f30a791414f0625b + fdc20074cf1e48b8cf11fe6ac78f255b1fbfe611 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index abe1718993c8..929b85c5458f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,12 +11,13 @@ - 8.0.128 + 8.0.422 + 8.0.400 true release - rtm + preview rtm servicing @@ -30,10 +31,11 @@ 2.0.1-servicing-26011-01 2.0.3 13.0.3 + 4.8.6 1.2.0-beta.435 - 7.0.0 + 8.0.0 4.0.0 - 7.0.0 + 8.0.1 8.0.0-beta.26224.3 7.0.0-preview.22423.2 8.0.0 @@ -57,6 +59,7 @@ 8.0.27 8.0.27-servicing.26229.22 8.0.0 + $(MicrosoftExtensionsDependencyModelPackageVersion) 8.0.1 8.0.3 8.0.1 @@ -72,7 +75,7 @@ 8.0.0 8.0.2 8.0.1 - 8.0.0 + 8.0.7 8.0.1 8.0.3 8.0.0 @@ -85,26 +88,26 @@ - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 - 6.8.2-rc.3 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 + 6.11.2-rc.1 $(NuGetPackagingPackageVersion) $(NuGetProjectModelPackageVersion) - 17.8.0-release-23615-02 - 17.8.0-release-23615-02 - 17.8.0-release-23615-02 + 17.11.1-release-24455-02 + 17.11.1-release-24455-02 + 17.11.1-release-24455-02 @@ -114,58 +117,56 @@ - 8.1.657407 + 8.3.657409 - 8.0.0-preview.26255.1 - 3.11.0-beta1.26255.1 + 8.0.0-preview.23614.1 + 3.11.0-beta1.23614.1 - 17.8.49 + 17.11.48 $(MicrosoftBuildPackageVersion) - $(MicrosoftBuildPackageVersion) - $(MicrosoftBuildPackageVersion) - 17.8.49-servicing-25616-08 - $(MicrosoftBuildPackageVersion) - $(MicrosoftBuildPackageVersion) + This avoids the need to juggle references to packages that have been updated in newer MSBuild. --> + 17.8.43 + $(MicrosoftBuildMinimumVersion) + $(MicrosoftBuildMinimumVersion) + 17.11.48-servicing-25466-05 + $(MicrosoftBuildMinimumVersion) + $(MicrosoftBuildMinimumVersion) $(MicrosoftBuildTasksCorePackageVersion) + $(MicrosoftBuildMinimumVersion) - 8.0.128 + 8.0.422 $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 8.0.128-servicing.26267.1 + 8.0.422-servicing.26256.3 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) - 12.8.102-beta.24081.2 + 12.8.403-beta.24526.2 - 4.8.0-7.25569.25 - 4.8.0-7.25569.25 - 4.8.0-7.25569.25 - 4.8.0-7.25569.25 - 4.8.0-7.25569.25 - 4.8.0-7.25569.25 - 4.8.0-7.25569.25 + 4.11.0-3.25569.22 + 4.11.0-3.25569.22 + 4.11.0-3.25569.22 + 4.11.0-3.25569.22 + 4.11.0-3.25569.22 + 4.11.0-3.25569.22 + 4.11.0-3.25569.22 $(MicrosoftNetCompilersToolsetPackageVersion) @@ -180,9 +181,9 @@ - 7.0.0-preview.26064.3 - 7.0.0-preview.26064.3 - 7.0.0-preview.26064.3 + 9.0.0-preview.26227.1 + 9.0.0-preview.26227.1 + 9.0.0-preview.26227.1 From dc9f730df563486b79657bf3a4e3600eff8c1f0d Mon Sep 17 00:00:00 2001 From: ".NET Source-Build Bot" <102560831+dotnet-sb-bot@users.noreply.github.com> Date: Mon, 18 May 2026 10:17:54 -0500 Subject: [PATCH 15/29] .NET Source-Build 9.0.117 May 2026 Updates (#54273) Co-authored-by: Nikola Milosavljevic --- src/SourceBuild/content/eng/Version.Details.xml | 4 ++-- src/SourceBuild/content/eng/Versions.props | 4 ++-- src/SourceBuild/content/global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/SourceBuild/content/eng/Version.Details.xml b/src/SourceBuild/content/eng/Version.Details.xml index 44d2fd58d811..0d713ed76339 100644 --- a/src/SourceBuild/content/eng/Version.Details.xml +++ b/src/SourceBuild/content/eng/Version.Details.xml @@ -2,9 +2,9 @@ - + https://github.com/dotnet/arcade - 01c94b66e75697e9298eb79a49db74489f8d343e + e1240639569fad610705b52713d6d6b19f8fe433 diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index cea15caeecd6..1705ce9fad61 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -23,8 +23,8 @@ of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts are not necessary, and this property is removed from the file. --> - 9.0.116 - 9.0.116-servicing.26177.1 + 9.0.117 + 9.0.117-servicing.26230.1 2.0.0-beta4.24126.1 diff --git a/src/SourceBuild/content/global.json b/src/SourceBuild/content/global.json index 352ca7311ba2..39645861c494 100644 --- a/src/SourceBuild/content/global.json +++ b/src/SourceBuild/content/global.json @@ -1,10 +1,10 @@ { "tools": { - "dotnet": "9.0.116" + "dotnet": "9.0.117" }, "msbuild-sdks": { "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26152.3" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26201.6" } } From 2a72bcc6c9cad22923f1206e4081c3149f67525c Mon Sep 17 00:00:00 2001 From: Missy Messa Date: Mon, 18 May 2026 21:47:20 +0000 Subject: [PATCH 16/29] Merged PR 61246: Use WIF federated token for internal VMR push Replace expired PAT `$(dn-bot-dnceng-build-rw-code-rw)` with the federated access token `$(AzdoToken)` already minted by the `VmrSyncPipeline` service connection earlier in the pipeline.\n\nThe `get-federated-access-token.yml` step already runs for all internal builds and produces `$(AzdoToken)` via the `VmrSyncPipeline` WIF service connection. The internal VMR push step was still referencing the expired PAT variable instead of using this token.\n\nFixes: AB#10724 ---- #### AI description (iteration 1) #### PR Classification Authentication configuration update to use Workload Identity Federation (WIF) federated token instead of personal access token for internal VMR repository push operations. #### PR Summary This pull request updates the authentication mechanism for pushing changes to the internal dotnet-dotnet VMR repository by replacing the static personal access token with a dynamically generated federated token. - `/eng/pipelines/templates/jobs/vmr-synchronization.yml`: Changed the `--azdev-pat` parameter from `$(dn-bot-dnceng-build-rw-code-rw)` to `$(AzdoToken)` to use WIF federated token for Azure DevOps authentication Related work items: #10724 --- eng/pipelines/templates/jobs/vmr-synchronization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/vmr-synchronization.yml b/eng/pipelines/templates/jobs/vmr-synchronization.yml index 3f3c7687b6a4..94d3331e2f62 100644 --- a/eng/pipelines/templates/jobs/vmr-synchronization.yml +++ b/eng/pipelines/templates/jobs/vmr-synchronization.yml @@ -87,7 +87,7 @@ jobs: --skip-commit-verification --branch '${{ parameters.vmrBranch }}' --remote-url '$(vmrInternalUrl)' - --azdev-pat '$(dn-bot-dnceng-build-rw-code-rw)' + --azdev-pat '$(AzdoToken)' --verbose displayName: Push changes to dotnet-dotnet (internal) workingDirectory: $(Agent.BuildDirectory)/sdk From cf8ea384e5d2aacd201a9f55d81c16d490d15171 Mon Sep 17 00:00:00 2001 From: ProductConstructionServiceProd Date: Mon, 18 May 2026 22:49:07 +0000 Subject: [PATCH 17/29] Merged PR 61256: [internal/release/9.0.3xx] Update dependencies from 3 repositories This pull request updates the following dependencies [marker]: <> (Begin:Coherency Updates) ## Coherency Updates The following updates ensure that dependencies with a *CoherentParentDependency* attribute were produced in a build used as input to the parent dependency's build. See [Dependency Description Format](https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md#dependency-description-overview) [DependencyUpdate]: <> (Begin) - **Coherency Updates**: - **Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100**: from 9.0.16 to 9.0.17 (parent: Microsoft.NETCore.App.Runtime.win-x64) - **Microsoft.SourceBuild.Intermediate.emsdk**: from 9.0.16-servicing.26221.3 to 9.0.17-servicing.26258.3 (parent: Microsoft.NETCore.App.Runtime.win-x64) [DependencyUpdate]: <> (End) [marker]: <> (End:Coherency Updates) [marker]: <> (Begin:fc735b81-9fd6-4d18-b1be-525bf9fe3d19) ## From https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - **Subscription**: [fc735b81-9fd6-4d18-b1be-525bf9fe3d19](https://maestro.dot.net/subscriptions?search=fc735b81-9fd6-4d18-b1be-525bf9fe3d19) - **Build**: [20260514.1](https://dev.azure.com/dnceng/internal/_build/results?buildId=2975597) ([314473](https://maestro.dot.net/channel/5128/azdo:dnceng:internal:dotnet-windowsdesktop/build/314473)) - **Date Produced**: May 15, 2026 6:29:00 AM UTC - **Commit**: [9d53c54d7228c41da31238e1e2442d67c486cb39](https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop?_a=history&version=GC9d53c54d7228c41da31238e1e2442d67c486cb39) - **Branch**: [refs/heads/internal/release/9.0](https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop?version=GBrefs/heads/internal/release/9.0) [DependencyUpdate]: <> (Begin) - **Dependency Updates**: - From [9.0.16 to 9.0.17][1] - Microsoft.WindowsDesktop.App.Ref - Microsoft.WindowsDesktop.App.Runtime.win-x64 - From [9.0.16-servicing.26230.2 to 9.0.17-servicing.26264.1][1] - VS.Redist.Common.WindowsDesktop.SharedFramework.x64.9.0 - VS.Redist.Common.WindowsDesktop.TargetingPack.x64.9.0 [1]: https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop/branches?baseVersion=GC7bad09c6a7b024bc98987a9fe2c66a79332bf8c5&targetVersion=GC9d53c54d7228c41da31238e1e2442d67c486cb39&_a=files [DependencyUpdate]: <> (End) [marker]: <> (End:fc735b81-9fd6-4d18-b1be-525bf9fe3d19) [marker]: <> (Begin:8e22721a-e9f6-419b-9c3b-fbb0f0641e55) ## From https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - **Subscription**: [8e22721a-e9f6-419b-9c3b-fbb0f0641e55](https://maestro.dot.net/subscriptions?search=8e22721a-e9f6-419b-9c3b-fbb0f0641e55) - **Build**: [20260514.16](https://dev.azure.com/dnceng/internal/_build/results?buildId=2975171) ([314446](https://maestro.dot.net/channel/5128/azdo:dnceng:internal:dotnet-runtime/build/314446)) - **Date Produced**: May 15, 2026 12:09:10 AM UTC - **Commit**: [f2c8152eed158e72950025393fde498c90a57a6b](https://dev.azure.com/dnceng/internal/... --- NuGet.config | 14 +-- eng/Version.Details.xml | 252 ++++++++++++++++++++-------------------- eng/Versions.props | 110 +++++++++--------- 3 files changed, 188 insertions(+), 188 deletions(-) diff --git a/NuGet.config b/NuGet.config index c67cde28f6d9..ddd5e51959e6 100644 --- a/NuGet.config +++ b/NuGet.config @@ -24,10 +24,10 @@ - + - + @@ -37,13 +37,13 @@ - + - + @@ -72,13 +72,13 @@ - + - + - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ebcaa794ba1c..6382ea470fcc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -15,42 +15,42 @@ b73682307aa0128c5edbec94c2e6a070d13ae6bb - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b @@ -59,14 +59,14 @@ https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - + https://github.com/dotnet/emsdk - e2909c00ead6fb5d18a5167ca78f259c639084e0 + b634e009d59f72e9254f984a6b89e685955e0eb8 - + https://github.com/dotnet/emsdk - e2909c00ead6fb5d18a5167ca78f259c639084e0 + b634e009d59f72e9254f984a6b89e685955e0eb8 @@ -131,13 +131,13 @@ https://github.com/dotnet/roslyn 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted @@ -226,99 +226,99 @@ 51441adcd6c424ae7315d66ce7e96baf34d70369 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 + 9d53c54d7228c41da31238e1e2442d67c486cb39 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 + 9d53c54d7228c41da31238e1e2442d67c486cb39 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 + 9d53c54d7228c41da31238e1e2442d67c486cb39 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 + 9d53c54d7228c41da31238e1e2442d67c486cb39 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - e5fdb70960f8d546f4122bbb4372e04a5031f60a + cd23381f043242e069217b1b1a65c141b5552e99 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 @@ -339,30 +339,30 @@ 41f3afd466695ac2460260431537fe4d779ff446 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - 639600f07395bb9bcdabbea5194ca0afaf753775 + 3fdbe05bf205479115e33916cdb200445b0a34e0 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - e5fdb70960f8d546f4122bbb4372e04a5031f60a + cd23381f043242e069217b1b1a65c141b5552e99 https://github.com/dotnet/xdt @@ -469,89 +469,89 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4d825aeb5e5023588c036709c7914008b625b0eb + 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b @@ -585,9 +585,9 @@ 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a1e6809fb8318884882ceff057000654f558738a + f2c8152eed158e72950025393fde498c90a57a6b https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index 51cbcfab1cd2..e679c05313b0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -81,61 +81,61 @@ - 9.0.16-servicing.26230.2 + 9.0.17-servicing.26264.2 - 9.0.16 - 9.0.16-servicing.26229.23 - 9.0.16 - 9.0.16 - 9.0.16-servicing.26229.23 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 + 9.0.17 + 9.0.17-servicing.26264.16 + 9.0.17 + 9.0.17 + 9.0.17-servicing.26264.16 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 8.0.0-rc.1.23414.4 - 9.0.16-servicing.26229.23 - 9.0.16-servicing.26229.23 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 + 9.0.17-servicing.26264.16 + 9.0.17-servicing.26264.16 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 2.1.0 - 9.0.16 + 9.0.17 8.0.0 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 8.0.0 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 - 9.0.16 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 + 9.0.17 4.5.1 4.5.5 8.0.5 4.5.4 - 9.0.16 - 9.0.16 + 9.0.17 + 9.0.17 - 9.0.16-servicing.26230.2 - 9.0.16-servicing.26230.2 - 9.0.16 - 9.0.16 + 9.0.17-servicing.26264.1 + 9.0.17-servicing.26264.1 + 9.0.17 + 9.0.17 @@ -217,19 +217,19 @@ - 9.0.16 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16 - 9.0.16 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 - 9.0.16-servicing.26230.9 + 9.0.17 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17 + 9.0.17 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 + 9.0.17-servicing.26268.2 @@ -239,8 +239,8 @@ - 9.0.16-rtm.26230.4 - 9.0.16-rtm.26230.4 + 9.0.17-rtm.26264.8 + 9.0.17-rtm.26264.8 @@ -319,7 +319,7 @@ 15.0.9617 18.0.9617 - 9.0.16 + 9.0.17 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100PackageVersion) 9.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-(?!rtm)[A-z]*[\.]*\d*`)) From 1bb3d3383e6ca159041375354d8f1eb360ecace3 Mon Sep 17 00:00:00 2001 From: ProductConstructionServiceProd Date: Mon, 18 May 2026 23:43:50 +0000 Subject: [PATCH 18/29] Merged PR 61260: [internal/release/9.0.3xx] Update dependencies from dnceng/internal/dotnet-windowsdesktop This pull request updates the following dependencies [marker]: <> (Begin:Coherency Updates) ## Coherency Updates The following updates ensure that dependencies with a *CoherentParentDependency* attribute were produced in a build used as input to the parent dependency's build. See [Dependency Description Format](https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md#dependency-description-overview) [DependencyUpdate]: <> (Begin) - **Coherency Updates**: - **Microsoft.NET.Sdk.WindowsDesktop**: from 9.0.17-rtm.26264.8 to 9.0.17-rtm.26267.1 (parent: Microsoft.WindowsDesktop.App.Ref) - **Microsoft.Dotnet.WinForms.ProjectTemplates**: from 9.0.17-servicing.26264.2 to 9.0.17-servicing.26265.1 (parent: Microsoft.WindowsDesktop.App.Runtime.win-x64) - **Microsoft.DotNet.Wpf.ProjectTemplates**: from 9.0.17-rtm.26264.8 to 9.0.17-rtm.26267.1 (parent: Microsoft.WindowsDesktop.App.Runtime.win-x64) [DependencyUpdate]: <> (End) [marker]: <> (End:Coherency Updates) [marker]: <> (Begin:fc735b81-9fd6-4d18-b1be-525bf9fe3d19) ## From https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - **Subscription**: [fc735b81-9fd6-4d18-b1be-525bf9fe3d19](https://maestro.dot.net/subscriptions?search=fc735b81-9fd6-4d18-b1be-525bf9fe3d19) - **Build**: [20260518.1](https://dev.azure.com/dnceng/internal/_build/results?buildId=2978264) ([314888](https://maestro.dot.net/channel/5128/azdo:dnceng:internal:dotnet-windowsdesktop/build/314888)) - **Date Produced**: May 18, 2026 11:19:48 PM UTC - **Commit**: [9c74c8784a90d64bcc5c5c552d5420eafb60fa4f](https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop?_a=history&version=GC9c74c8784a90d64bcc5c5c552d5420eafb60fa4f) - **Branch**: [refs/heads/internal/release/9.0](https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop?version=GBrefs/heads/internal/release/9.0) [DependencyUpdate]: <> (Begin) - **Dependency Updates**: - From [9.0.17 to 9.0.17][1] - Microsoft.WindowsDesktop.App.Ref - Microsoft.WindowsDesktop.App.Runtime.win-x64 - From [9.0.17-servicing.26264.1 to 9.0.17-servicing.26268.1][1] - VS.Redist.Common.WindowsDesktop.SharedFramework.x64.9.0 - VS.Redist.Common.WindowsDesktop.TargetingPack.x64.9.0 [1]: https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop/branches?baseVersion=GC9d53c54d7228c41da31238e1e2442d67c486cb39&targetVersion=GC9c74c8784a90d64bcc5c5c552d5420eafb60fa4f&_a=files [DependencyUpdate]: <> (End) [marker]: <> (End:fc735b81-9fd6-4d18-b1be-525bf9fe3d19) --- NuGet.config | 4 ++-- eng/Version.Details.xml | 24 ++++++++++++------------ eng/Versions.props | 10 +++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/NuGet.config b/NuGet.config index ddd5e51959e6..24dacdb6230c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -43,7 +43,7 @@ - + @@ -78,7 +78,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6382ea470fcc..5aebec0e1f6f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -252,24 +252,24 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9d53c54d7228c41da31238e1e2442d67c486cb39 + 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9d53c54d7228c41da31238e1e2442d67c486cb39 + 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9d53c54d7228c41da31238e1e2442d67c486cb39 + 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9d53c54d7228c41da31238e1e2442d67c486cb39 + 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - cd23381f043242e069217b1b1a65c141b5552e99 + a865df6e83bf8b34bbd6802fdcf25c5cdfb05191 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -356,13 +356,13 @@ 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - 3fdbe05bf205479115e33916cdb200445b0a34e0 + 4569ad0b3bd84191660c74228ad6d18d29fbc1cb - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - cd23381f043242e069217b1b1a65c141b5552e99 + a865df6e83bf8b34bbd6802fdcf25c5cdfb05191 https://github.com/dotnet/xdt diff --git a/eng/Versions.props b/eng/Versions.props index e679c05313b0..241868f446e1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -81,7 +81,7 @@ - 9.0.17-servicing.26264.2 + 9.0.17-servicing.26265.1 @@ -132,8 +132,8 @@ - 9.0.17-servicing.26264.1 - 9.0.17-servicing.26264.1 + 9.0.17-servicing.26268.1 + 9.0.17-servicing.26268.1 9.0.17 9.0.17 @@ -239,8 +239,8 @@ - 9.0.17-rtm.26264.8 - 9.0.17-rtm.26264.8 + 9.0.17-rtm.26267.1 + 9.0.17-rtm.26267.1 From 38a909ff4b1011fd283d2919d6cbabf5d895087e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 19 May 2026 02:01:57 +0000 Subject: [PATCH 19/29] Update dependencies from https://github.com/dotnet/templating build 20260517.3 On relative base path root Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Mocks From Version 8.0.422-servicing.26256.3 -> To Version 8.0.422-servicing.26267.3 Microsoft.TemplateEngine.Abstractions From Version 8.0.422 -> To Version 8.0.422 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 2dd596731770..45aec750d9f9 100644 --- a/NuGet.config +++ b/NuGet.config @@ -17,7 +17,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4cc343bef588..198f8725576c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,15 +3,15 @@ https://github.com/dotnet/templating - 3dba2c56d483d784cf7241e2fef06b7f6c845d96 + 7fa69a13441d30db86cec3bf36aa68d72fbe04bf - + https://github.com/dotnet/templating - 3dba2c56d483d784cf7241e2fef06b7f6c845d96 + 7fa69a13441d30db86cec3bf36aa68d72fbe04bf - + https://github.com/dotnet/templating - 3dba2c56d483d784cf7241e2fef06b7f6c845d96 + 7fa69a13441d30db86cec3bf36aa68d72fbe04bf diff --git a/eng/Versions.props b/eng/Versions.props index 929b85c5458f..2d9ea0fe47d3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -149,7 +149,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 8.0.422-servicing.26256.3 + 8.0.422-servicing.26267.3 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 40256c11fe323d5074d73114f6a5fa4e39353927 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 04:11:47 +0000 Subject: [PATCH 20/29] Reset files to release/9.0.1xx Reset patterns: - global.json - NuGet.config - eng/Version.Details.xml - eng/Versions.props - eng/common/* --- NuGet.config | 38 +- eng/Version.Details.xml | 673 +++++++++++------- eng/Versions.props | 375 ++++++---- eng/common/SetupNugetSources.ps1 | 54 +- eng/common/SetupNugetSources.sh | 62 +- eng/common/build.cmd | 3 + eng/common/build.ps1 | 3 + eng/common/build.sh | 15 +- eng/common/core-templates/job/job.yml | 253 +++++++ eng/common/core-templates/job/onelocbuild.yml | 121 ++++ .../job/publish-build-assets.yml | 172 +++++ .../core-templates/job/source-build.yml | 99 +++ .../job/source-index-stage1.yml | 81 +++ .../core-templates/jobs/codeql-build.yml | 33 + eng/common/core-templates/jobs/jobs.yml | 123 ++++ .../core-templates/jobs/source-build.yml | 63 ++ .../post-build/common-variables.yml | 22 + .../core-templates/post-build/post-build.yml | 327 +++++++++ .../post-build/setup-maestro-vars.yml | 74 ++ .../steps/component-governance.yml | 16 + .../steps/enable-internal-runtimes.yml | 32 + .../steps/enable-internal-sources.yml | 47 ++ .../core-templates/steps/generate-sbom.yml | 54 ++ .../steps/get-delegation-sas.yml | 55 ++ .../steps/get-federated-access-token.yml | 42 ++ .../steps/publish-build-artifacts.yml | 20 + .../core-templates/steps/publish-logs.yml | 58 ++ .../steps/publish-pipeline-artifacts.yml | 20 + .../core-templates/steps/retain-build.yml | 28 + .../core-templates/steps/send-to-helix.yml | 93 +++ .../core-templates/steps/source-build.yml | 137 ++++ .../variables/pool-providers.yml | 8 + eng/common/cross/build-android-rootfs.sh | 8 +- eng/common/cross/build-rootfs.sh | 262 +++++-- eng/common/cross/riscv64/tizen/tizen.patch | 9 + eng/common/cross/tizen-build-rootfs.sh | 21 + eng/common/cross/tizen-fetch.sh | 19 +- eng/common/cross/toolchain.cmake | 25 +- eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 2 +- eng/common/dotnet-install.sh | 5 +- eng/common/generate-sbom-prep.sh | 2 +- eng/common/helixpublish.proj | 1 + eng/common/internal/Directory.Build.props | 7 + eng/common/internal/NuGet.config | 3 + eng/common/internal/Tools.csproj | 14 +- eng/common/native/CommonLibrary.psm1 | 3 +- eng/common/native/init-compiler.sh | 117 +-- eng/common/native/init-distro-rid.sh | 76 +- eng/common/native/init-os-and-arch.sh | 7 +- .../post-build/check-channel-consistency.ps1 | 12 +- eng/common/post-build/nuget-validation.ps1 | 9 +- eng/common/post-build/nuget-verification.ps1 | 2 +- eng/common/post-build/publish-using-darc.ps1 | 15 +- eng/common/post-build/redact-logs.ps1 | 89 +++ .../post-build/sourcelink-validation.ps1 | 10 +- eng/common/post-build/symbols-validation.ps1 | 2 - eng/common/sdl/NuGet.config | 4 +- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 +- eng/common/sdl/init-sdl.ps1 | 8 + eng/common/sdl/packages.config | 2 +- eng/common/sdl/sdl.ps1 | 4 +- eng/common/sdl/trim-assets-version.ps1 | 2 +- eng/common/template-guidance.md | 133 ++++ eng/common/templates-official/job/job.yml | 340 ++------- .../templates-official/job/onelocbuild.yml | 115 +-- .../job/publish-build-assets.yml | 180 +---- .../templates-official/job/source-build.yml | 82 +-- .../job/source-index-stage1.yml | 86 +-- .../templates-official/jobs/codeql-build.yml | 32 +- eng/common/templates-official/jobs/jobs.yml | 104 +-- .../templates-official/jobs/source-build.yml | 62 +- .../post-build/common-variables.yml | 26 +- .../post-build/post-build.yml | 295 +------- .../post-build/setup-maestro-vars.yml | 74 +- .../steps/component-governance.yml | 18 +- .../steps/enable-internal-runtimes.yml | 31 +- .../steps/enable-internal-sources.yml | 7 + .../steps/generate-sbom.yml | 51 +- .../steps/get-delegation-sas.yml | 55 +- .../steps/get-federated-access-token.yml | 45 +- .../steps/publish-build-artifacts.yml | 41 ++ .../templates-official/steps/publish-logs.yml | 26 +- .../steps/publish-pipeline-artifacts.yml | 28 + .../templates-official/steps/retain-build.yml | 33 +- .../steps/send-to-helix.yml | 97 +-- .../templates-official/steps/source-build.yml | 138 +--- eng/common/templates/job/job.yml | 319 ++------- eng/common/templates/job/onelocbuild.yml | 112 +-- .../templates/job/publish-build-assets.yml | 176 +---- eng/common/templates/job/source-build.yml | 81 +-- .../templates/job/source-index-stage1.yml | 85 +-- eng/common/templates/jobs/codeql-build.yml | 32 +- eng/common/templates/jobs/jobs.yml | 104 +-- eng/common/templates/jobs/source-build.yml | 62 +- .../templates/post-build/common-variables.yml | 26 +- .../templates/post-build/post-build.yml | 291 +------- .../post-build/setup-maestro-vars.yml | 74 +- .../templates/steps/component-governance.yml | 18 +- .../steps/enable-internal-runtimes.yml | 30 +- .../steps/enable-internal-sources.yml | 7 + eng/common/templates/steps/generate-sbom.yml | 51 +- .../templates/steps/get-delegation-sas.yml | 55 +- .../steps/get-federated-access-token.yml | 45 +- .../steps/publish-build-artifacts.yml | 40 ++ eng/common/templates/steps/publish-logs.yml | 26 +- .../steps/publish-pipeline-artifacts.yml | 34 + eng/common/templates/steps/retain-build.yml | 33 +- eng/common/templates/steps/send-to-helix.yml | 97 +-- eng/common/templates/steps/source-build.yml | 138 +--- .../templates/variables/pool-providers.yml | 54 +- eng/common/tools.ps1 | 60 +- eng/common/tools.sh | 39 +- global.json | 13 +- 114 files changed, 3991 insertions(+), 4157 deletions(-) create mode 100644 eng/common/build.cmd create mode 100644 eng/common/core-templates/job/job.yml create mode 100644 eng/common/core-templates/job/onelocbuild.yml create mode 100644 eng/common/core-templates/job/publish-build-assets.yml create mode 100644 eng/common/core-templates/job/source-build.yml create mode 100644 eng/common/core-templates/job/source-index-stage1.yml create mode 100644 eng/common/core-templates/jobs/codeql-build.yml create mode 100644 eng/common/core-templates/jobs/jobs.yml create mode 100644 eng/common/core-templates/jobs/source-build.yml create mode 100644 eng/common/core-templates/post-build/common-variables.yml create mode 100644 eng/common/core-templates/post-build/post-build.yml create mode 100644 eng/common/core-templates/post-build/setup-maestro-vars.yml create mode 100644 eng/common/core-templates/steps/component-governance.yml create mode 100644 eng/common/core-templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/core-templates/steps/enable-internal-sources.yml create mode 100644 eng/common/core-templates/steps/generate-sbom.yml create mode 100644 eng/common/core-templates/steps/get-delegation-sas.yml create mode 100644 eng/common/core-templates/steps/get-federated-access-token.yml create mode 100644 eng/common/core-templates/steps/publish-build-artifacts.yml create mode 100644 eng/common/core-templates/steps/publish-logs.yml create mode 100644 eng/common/core-templates/steps/publish-pipeline-artifacts.yml create mode 100644 eng/common/core-templates/steps/retain-build.yml create mode 100644 eng/common/core-templates/steps/send-to-helix.yml create mode 100644 eng/common/core-templates/steps/source-build.yml create mode 100644 eng/common/core-templates/variables/pool-providers.yml create mode 100644 eng/common/cross/riscv64/tizen/tizen.patch create mode 100644 eng/common/post-build/redact-logs.ps1 create mode 100644 eng/common/template-guidance.md create mode 100644 eng/common/templates-official/steps/enable-internal-sources.yml create mode 100644 eng/common/templates-official/steps/publish-build-artifacts.yml create mode 100644 eng/common/templates-official/steps/publish-pipeline-artifacts.yml create mode 100644 eng/common/templates/steps/enable-internal-sources.yml create mode 100644 eng/common/templates/steps/publish-build-artifacts.yml create mode 100644 eng/common/templates/steps/publish-pipeline-artifacts.yml diff --git a/NuGet.config b/NuGet.config index 45aec750d9f9..dcf574945f80 100644 --- a/NuGet.config +++ b/NuGet.config @@ -3,26 +3,42 @@ + + + + + + + + + + + + + + + + + + + + - - - + - + - - @@ -30,6 +46,9 @@ + + + @@ -39,23 +58,18 @@ - - - - + - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 198f8725576c..914fc315ca9c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,521 +1,664 @@ - + https://github.com/dotnet/templating - 7fa69a13441d30db86cec3bf36aa68d72fbe04bf + c96cefdd04b806246127b9ad69dcb60c31b56e6f - + https://github.com/dotnet/templating - 7fa69a13441d30db86cec3bf36aa68d72fbe04bf + c96cefdd04b806246127b9ad69dcb60c31b56e6f - + + https://github.com/dotnet/templating - 7fa69a13441d30db86cec3bf36aa68d72fbe04bf + c96cefdd04b806246127b9ad69dcb60c31b56e6f - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 - + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + a1e6809fb8318884882ceff057000654f558738a - + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a + + + + + + https://github.com/dotnet/core-setup + 7d57652f33493fa022125b7f63aad0d70c52d810 + + + https://github.com/dotnet/emsdk + e2909c00ead6fb5d18a5167ca78f259c639084e0 + + + https://github.com/dotnet/emsdk + e2909c00ead6fb5d18a5167ca78f259c639084e0 - + + https://github.com/dotnet/emsdk - 925a4f472f7e1c6d3bcb5092d28d952f90cc6237 + e2909c00ead6fb5d18a5167ca78f259c639084e0 + - - https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted - 02bf66295b64ab368d12933041f7281aad186a2d + + https://github.com/dotnet/msbuild + 07da1b9a89da6d00c5a5a6a385cdcebfdfed7110 - - https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted - 02bf66295b64ab368d12933041f7281aad186a2d + + https://github.com/dotnet/msbuild + 07da1b9a89da6d00c5a5a6a385cdcebfdfed7110 - - https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted - 02bf66295b64ab368d12933041f7281aad186a2d + + + https://github.com/dotnet/msbuild + 07da1b9a89da6d00c5a5a6a385cdcebfdfed7110 - + https://github.com/dotnet/fsharp - e11d7079bebc6f101c5313fe0d1de9e3d38a7c02 + 47d4e3f91e4e5414b6dafbf14288b9c5a798ef99 - + + https://github.com/dotnet/fsharp - e11d7079bebc6f101c5313fe0d1de9e3d38a7c02 + 47d4e3f91e4e5414b6dafbf14288b9c5a798ef99 - - https://github.com/dotnet/format - b7d483d5a0ce4e96a3468514f3f4a98f3c371434 - + + https://github.com/dotnet/roslyn + fc52718eccdb37693a40a518b1178b1e23114e68 - + + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + + https://github.com/dotnet/roslyn + fc52718eccdb37693a40a518b1178b1e23114e68 + + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + https://github.com/dotnet/roslyn - 3fb752d448006a3144a60ccf181d745e555422f9 + fc52718eccdb37693a40a518b1178b1e23114e68 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d + - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c754078ce0c9bba2fc03aa6fd45c1fd760c25a62 + c92125011405028c945371a89e1a1eb0e735456d - + https://github.com/microsoft/vstest - 7855c9b221686104532ebf3380f2d45b3613b369 - + bc9161306b23641b0364b8f93d546da4d48da1eb - + https://github.com/microsoft/vstest - 7855c9b221686104532ebf3380f2d45b3613b369 + bc9161306b23641b0364b8f93d546da4d48da1eb - + https://github.com/microsoft/vstest - 7855c9b221686104532ebf3380f2d45b3613b369 + bc9161306b23641b0364b8f93d546da4d48da1eb - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + + + https://github.com/microsoft/vstest + bc9161306b23641b0364b8f93d546da4d48da1eb + - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 50c4cb9fc31c47f03eac865d7bc518af173b74b7 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 50c4cb9fc31c47f03eac865d7bc518af173b74b7 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - f219383822edeeaf58b32c4808576cf4b159ba71 + 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 + - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - f219383822edeeaf58b32c4808576cf4b159ba71 + 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - f219383822edeeaf58b32c4808576cf4b159ba71 + 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - f219383822edeeaf58b32c4808576cf4b159ba71 + 7bad09c6a7b024bc98987a9fe2c66a79332bf8c5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - 30f6a082ed1780e12e452f40374f0d95d7082383 + e5fdb70960f8d546f4122bbb4372e04a5031f60a - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d - + 4d825aeb5e5023588c036709c7914008b625b0eb + + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb + - + https://github.com/dotnet/razor - 2f31e7f15e35ce060c1ba81174a87fdd0ae9f479 - + cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 - + https://github.com/dotnet/razor - 2f31e7f15e35ce060c1ba81174a87fdd0ae9f479 + cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 - + https://github.com/dotnet/razor - 2f31e7f15e35ce060c1ba81174a87fdd0ae9f479 + cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 - + + + https://github.com/dotnet/razor + cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 + + + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb + + + https://github.com/dotnet/test-templates + 0385265f4d0b6413d64aea0223172366a9b9858c + + + https://github.com/dotnet/test-templates + 307b8f538d83a955d8f6dd909eee41a5555f2f4d + + + https://github.com/dotnet/test-templates + becc4bd157cd6608b51a5ffe414a5d2de6330272 + + + https://github.com/dotnet/test-templates + becc4bd157cd6608b51a5ffe414a5d2de6330272 + + + https://github.com/dotnet/test-templates + 49c9ad01f057b3c6352bbec12b117acc2224493c - + + https://github.com/dotnet/test-templates + 47c90e140b027225b799ca8413af10ee3d5f1126 + + + + https://github.com/dotnet/test-templates + 47c90e140b027225b799ca8413af10ee3d5f1126 + + + + + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms + 639600f07395bb9bcdabbea5194ca0afaf753775 + + + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf + e5fdb70960f8d546f4122bbb4372e04a5031f60a + + https://github.com/dotnet/xdt - 9a1c3e1b7f0c8763d4c96e593961a61a72679a7b + 63ae81154c50a1cf9287cc47d8351d55b4289e6d + + + + https://github.com/dotnet/xdt + 63ae81154c50a1cf9287cc47d8351d55b4289e6d - + https://github.com/dotnet/roslyn-analyzers - abef8ced132657943b7150f01a308e2199a17d5d + b9b54526b7908ea519b503196100a34dd2e52374 - + https://github.com/dotnet/roslyn-analyzers - abef8ced132657943b7150f01a308e2199a17d5d + b9b54526b7908ea519b503196100a34dd2e52374 - + + https://github.com/dotnet/roslyn-analyzers - abef8ced132657943b7150f01a308e2199a17d5d + b9b54526b7908ea519b503196100a34dd2e52374 - + + https://github.com/dotnet/command-line-api + 803d8598f98fb4efd94604b32627ee9407f246db + + https://github.com/dotnet/command-line-api - 02fe27cd6a9b001c8feb7938e6ef4b3799745759 + 803d8598f98fb4efd94604b32627ee9407f246db - + + + + https://github.com/dotnet/symreader + 0710a7892d89999956e8808c28e9dd0512bd53f3 + + + https://github.com/dotnet/command-line-api - 02fe27cd6a9b001c8feb7938e6ef4b3799745759 + 803d8598f98fb4efd94604b32627ee9407f246db - + + https://github.com/dotnet/source-build-externals - 068e37394c61ba439d2898151c6c078dad3ef66b + 16c380d1ce5fa0b24e232251c31cb013bbf3365f - - https://github.com/dotnet/source-build-reference-packages - 9d83c59ff890ff2020aa38ebcf3bb514e38e9ffe - + + + https://github.com/dotnet/source-build-assets + 3a687f7154d73a6425d98a1deeb40424bbd7cb38 + - + https://github.com/dotnet/deployment-tools - 5255d40e228ea1d4b624781b5b97ec16484a3b4b + b2d5c0c5841de4bc036ef4c84b5db3532504e5f3 - + https://github.com/dotnet/sourcelink - 94eaac3385cafff41094454966e1af1d1cf60f00 - + 657ade4711e607cc4759e89e0943aa1ca8aadc63 - + https://github.com/dotnet/sourcelink - 94eaac3385cafff41094454966e1af1d1cf60f00 + 657ade4711e607cc4759e89e0943aa1ca8aadc63 - + https://github.com/dotnet/sourcelink - 94eaac3385cafff41094454966e1af1d1cf60f00 + 657ade4711e607cc4759e89e0943aa1ca8aadc63 - + https://github.com/dotnet/sourcelink - 94eaac3385cafff41094454966e1af1d1cf60f00 + 657ade4711e607cc4759e89e0943aa1ca8aadc63 - + https://github.com/dotnet/sourcelink - 94eaac3385cafff41094454966e1af1d1cf60f00 + 657ade4711e607cc4759e89e0943aa1ca8aadc63 - + https://github.com/dotnet/sourcelink - 94eaac3385cafff41094454966e1af1d1cf60f00 + 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - + + + https://github.com/dotnet/sourcelink + 657ade4711e607cc4759e89e0943aa1ca8aadc63 + + + + https://github.com/dotnet/deployment-tools - 5255d40e228ea1d4b624781b5b97ec16484a3b4b + b2d5c0c5841de4bc036ef4c84b5db3532504e5f3 - + + https://github.com/dotnet/symreader - 27e584661980ee6d82c419a2a471ae505b7d122e + 0710a7892d89999956e8808c28e9dd0512bd53f3 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 50c4cb9fc31c47f03eac865d7bc518af173b74b7 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - ef853a71052646a42abf17e888ec6d9a69614ad9 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - be2530c3035e4bfa7670c6b18f5a64ef89e0e80d + 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + a1e6809fb8318884882ceff057000654f558738a - - https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - fdc20074cf1e48b8cf11fe6ac78f255b1fbfe611 - - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - a6bde67c455f2ac219988c7a66171631090b6f65 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + a1e6809fb8318884882ceff057000654f558738a - + https://github.com/dotnet/arcade - d9af20b993c474033098fe0851c2d71b4ecf434b - + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 - + https://github.com/dotnet/arcade - d9af20b993c474033098fe0851c2d71b4ecf434b + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 - + https://github.com/dotnet/arcade - d9af20b993c474033098fe0851c2d71b4ecf434b + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 - + https://github.com/dotnet/arcade - d9af20b993c474033098fe0851c2d71b4ecf434b + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 81cabf2857a01351e5ab578947c7403a5b128ad1 + + https://github.com/dotnet/arcade + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + + + https://github.com/dotnet/arcade + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + + + + https://github.com/dotnet/arcade + f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + - - https://github.com/dotnet/xliff-tasks - 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime + a1e6809fb8318884882ceff057000654f558738a + + + https://github.com/dotnet/arcade-services + e156e649f28395d9d0ee1e848225a689b59e0fd3 + + + https://github.com/dotnet/arcade-services + e156e649f28395d9d0ee1e848225a689b59e0fd3 + + + https://github.com/dotnet/scenario-tests + d130a15593a5a26d1c35da3d4204e35bf99e1e9d + + + + https://github.com/dotnet/scenario-tests + d130a15593a5a26d1c35da3d4204e35bf99e1e9d + + + + + https://github.com/dotnet/aspire + 5fa9337a84a52e9bd185d04d156eccbdcf592f74 + + + + https://github.com/dotnet/aspire + 5fa9337a84a52e9bd185d04d156eccbdcf592f74 + https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index 2d9ea0fe47d3..457c83a51328 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,193 +1,267 @@ - - - + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) - - - true - true - - - - 8.0.422 - 8.0.400 + + 9 + 0 + 1 + 18 + + $(VersionMajor).$(VersionMinor).$(VersionSDKMinor)00 + $(VersionMajor).$(VersionMinor).$(VersionSDKMinor)$(VersionFeature) + $(VersionMajor).$(VersionMinor) + $(MajorMinorVersion).$(VersionSDKMinor) true release - preview + rtm rtm servicing + + true 6.0.1 - - + + 30 + 32 + 17 + 36 + 20 + $([MSBuild]::Add($(VersionFeature), 10)) + + <_NET70ILLinkPackVersion>7.0.100-1.23211.1 + + + + https://ci.dot.net/public/ + https://dotnetclimsrc.blob.core.windows.net/dotnet/ + + + 10.0.0-preview.24609.2 1.0.0-20230414.1 - 2.21.0 + 2.22.0 2.0.1-servicing-26011-01 2.0.3 13.0.3 4.8.6 1.2.0-beta.435 - 8.0.0 - 4.0.0 - 8.0.1 - 8.0.0-beta.26224.3 - 7.0.0-preview.22423.2 - 8.0.0 - 4.3.0 - 4.3.0 4.0.5 - 8.0.1 - 4.6.0 - 2.0.0-beta4.23307.1 - 2.0.0-rtm.1.25064.1 + 2.0.0-beta4.24324.3 + 0.4.0-alpha.24324.3 + 2.0.0-rtm.1.25059.4 + 2.2.0-beta.24327.2 + 1.1.2-beta1.22216.1 + 10.3.0 3.2.2146 0.3.49-beta + + + 1.8.1 + + + 0.2.0 + - - 8.0.27 - 8.0.27-servicing.26229.22 - 8.0.27 - $(MicrosoftNETCoreAppRuntimewinx64PackageVersion) - 8.0.2 - 8.0.27 - 8.0.27-servicing.26229.22 - 8.0.0 - $(MicrosoftExtensionsDependencyModelPackageVersion) - 8.0.1 - 8.0.3 - 8.0.1 - 8.0.27 - 8.0.0 - 8.0.0 - 8.0.27 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.2 - 8.0.1 - 8.0.7 - 8.0.1 - 8.0.3 - 8.0.0 - 8.0.1 - 8.0.6 - - 8.0.5 - 8.0.0 - 8.0.2 + + 1.1.0-beta.25317.4 - - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - 6.11.2-rc.1 - $(NuGetPackagingPackageVersion) - $(NuGetProjectModelPackageVersion) + + 9.0.16-servicing.26230.2 - - 17.11.1-release-24455-02 - 17.11.1-release-24455-02 - 17.11.1-release-24455-02 + + + 1.1.0-rtm.25262.1 - 8.0.0 - 8.0.0 - 8.0.0 + 9.0.16 + 9.0.16-servicing.26229.23 + 9.0.16 + 9.0.16 + 9.0.16-servicing.26229.23 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 8.0.0-rc.1.23414.4 + 9.0.16-servicing.26229.23 + 9.0.16-servicing.26229.23 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 2.1.0 + 9.0.16 + 8.0.0 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 8.0.0 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + 9.0.16 + + 8.0.5 + 9.0.16 + 9.0.16 - - 8.3.657409 + + 9.0.16-servicing.26230.2 + 9.0.16-servicing.26230.2 + 9.0.16 + 9.0.16 + + + + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + 6.12.5-rc.1 + + + + 17.12.0-release-24508-01 + 17.12.0-release-24508-01 + 17.12.0-release-24508-01 - 8.0.0-preview.23614.1 - 3.11.0-beta1.23614.1 + 9.0.0-preview.26226.5 + 3.11.0-beta1.26226.5 - 17.11.48 - $(MicrosoftBuildPackageVersion) - - 17.8.43 - $(MicrosoftBuildMinimumVersion) - $(MicrosoftBuildMinimumVersion) - 17.11.48-servicing-25466-05 - $(MicrosoftBuildMinimumVersion) - $(MicrosoftBuildMinimumVersion) - $(MicrosoftBuildTasksCorePackageVersion) - $(MicrosoftBuildMinimumVersion) + At usage sites, either we use MicrosoftBuildMinimumVersion, or MicrosoftBuildVersion in source-only modes. + + Additionally, set the MinimumVSVersion for the installer UI that's required for targeting NetCurrent --> + 17.12.57 + 17.12.57-preview-26069-01 + 17.11.48 + 17.12 - 8.0.422 + 9.0.118 $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 8.0.422-servicing.26267.3 + 9.0.118-servicing.26267.4 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) - 12.8.403-beta.24526.2 + 12.9.101-beta.25070.7 - 4.11.0-3.25569.22 - 4.11.0-3.25569.22 - 4.11.0-3.25569.22 - 4.11.0-3.25569.22 - 4.11.0-3.25569.22 - 4.11.0-3.25569.22 - 4.11.0-3.25569.22 - $(MicrosoftNetCompilersToolsetPackageVersion) + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 + 4.12.0-3.25609.5 - 8.0.27 - 8.0.27-servicing.26230.8 - 8.0.27-servicing.26230.8 - 8.0.27-servicing.26230.8 - 8.0.27-servicing.26230.8 - 8.0.27-servicing.26230.8 - 8.0.27 + 9.0.16 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16 + 9.0.16 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 + 9.0.16-servicing.26230.9 - - 9.0.0-preview.26227.1 - 9.0.0-preview.26227.1 - 9.0.0-preview.26227.1 + + 9.0.0-preview.26069.1 + 9.0.0-preview.26069.1 + 9.0.0-preview.26069.1 - 8.0.27-servicing.26230.2 + 9.0.16-rtm.26230.4 + 9.0.16-rtm.26230.4 + + + + $(MicrosoftNETCoreAppHostwinx64PackageVersion) + $(MicrosoftNETCoreAppRuntimewinx64PackageVersion) + $(MicrosoftAspNetCoreAppRuntimewinx64PackageVersion) + $(MicrosoftWindowsDesktopAppRuntimewinx64PackageVersion) + + + $(MicrosoftNETCoreAppRuntimePackageVersion) + $(MicrosoftNETCoreAppRuntimePackageVersion) + + + + $(MicrosoftAspNetCoreAppRuntimePackageVersion) + $(MicrosoftDotnetWinFormsProjectTemplatesPackageVersion) + $(MicrosoftDotNetWpfProjectTemplatesPackageVersion) + $(NUnit3DotNetNewTemplatePackageVersion) + + + 2.2.0-beta.19072.10 + 2.0.0 + 9.0.0-preview.26263.1 @@ -195,17 +269,20 @@ 4.0.1 - - 1.0.0-beta.23475.1 + + 9.0.0-beta.26220.3 + 9.0.0-beta.26220.3 + 9.0.0-beta.26220.3 + 9.0.0-beta.26220.3 - 8.0.0-beta.23615.1 - 8.0.0-beta.23615.1 - 8.0.0-beta.23615.1 - 8.0.0-beta.23615.1 - 8.0.0-beta.23615.1 - 8.0.0-beta.23615.1 + 9.0.0-beta.25564.5 + 9.0.0-beta.25564.5 + 9.0.0-beta.25564.5 + 9.0.0-beta.25564.5 + 9.0.0-beta.25564.5 + 9.0.0-beta.25564.5 @@ -215,8 +292,8 @@ 6.12.0 6.1.0 - 8.0.0-beta.26224.3 4.18.4 + 3.4.3 1.3.2 8.0.0-beta.23607.1 @@ -229,15 +306,35 @@ <_DotNetHiveRoot Condition="!HasTrailingSlash('$(_DotNetHiveRoot)')">$(_DotNetHiveRoot)/ $(_DotNetHiveRoot)dotnet$(ExeExtension) - + + 8.0.100 + 8.2.2 + 9.0.100 + 9.0.0 + 35.0.7 + 18.0.9617 + 18.0.9617 + 15.0.9617 + 18.0.9617 - 8.0.27 - $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100PackageVersion) + 9.0.16-servicing.26221.3 + 9.0.16 + $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100PackageVersion) - 8.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-rtm|-[A-z]*\.*\d*`)) + 9.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-(?!rtm)[A-z]*[\.]*\d*`)) $(MicrosoftNETCoreAppRefPackageVersion) - 8.0.100$([System.Text.RegularExpressions.Regex]::Match($(MonoWorkloadManifestVersion), `-rtm|-[A-z]*\.*\d*`)) + 9.0.100$([System.Text.RegularExpressions.Regex]::Match($(MonoWorkloadManifestVersion), `-(?!rtm)[A-z]*[\.]*\d*`)) + + + + 15.7.179 + 15.7.179 + + + + 2.0.1-servicing-26011-01 + diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 59b2d55e1a33..792b60b49d42 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,17 +1,10 @@ -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing +# See example call for this script below. # # - task: PowerShell@2 # displayName: Setup Private Feeds Credentials @@ -21,11 +14,18 @@ # arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token # env: # Token: $(dn-bot-dnceng-artifact-feeds-rw) +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) +# - The credential provider is installed. +# +# This logic is also abstracted into enable-internal-sources.yml. [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - [Parameter(Mandatory = $true)][string]$Password + $Password ) $ErrorActionPreference = "Stop" @@ -48,11 +48,17 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source function AddCredential($creds, $source, $username, $pwd) { + # If no cred supplied, don't do anything. + if (!$pwd) { + return; + } + # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -110,11 +116,6 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) { ExitWithExitCode 1 } -if (!$Password) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT' - ExitWithExitCode 1 -} - # Load NuGet.config $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName @@ -127,11 +128,14 @@ if ($sources -eq $null) { $doc.DocumentElement.AppendChild($sources) | Out-Null } -# Looks for a node. Create it if none is found. -$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") -if ($creds -eq $null) { - $creds = $doc.CreateElement("packageSourceCredentials") - $doc.DocumentElement.AppendChild($creds) | Out-Null +$creds = $null +if ($Password) { + # Looks for a node. Create it if none is found. + $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") + if ($creds -eq $null) { + $creds = $doc.CreateElement("packageSourceCredentials") + $doc.DocumentElement.AppendChild($creds) | Out-Null + } } # Check for disabledPackageSources; we'll enable any darc-int ones we find there @@ -153,7 +157,7 @@ if ($dotnet31Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } -$dotnetVersions = @('5','6','7','8') +$dotnetVersions = @('5','6','7','8','9') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; @@ -164,4 +168,4 @@ foreach ($dotnetVersion in $dotnetVersions) { } } -$doc.Save($filename) \ No newline at end of file +$doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index c0e7bbef21c4..facb415ca6ff 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,28 +1,27 @@ #!/usr/bin/env bash -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro's managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. -# -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing. +# See example call for this script below. # # - task: Bash@3 -# displayName: Setup Private Feeds Credentials +# displayName: Setup Internal Feeds # inputs: # filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(System.DefaultWorkingDirectory)/NuGet.config $Token +# arguments: $(System.DefaultWorkingDirectory)/NuGet.config # condition: ne(variables['Agent.OS'], 'Windows_NT') -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# - task: NuGetAuthenticate@1 +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) +# - The credential provider is installed. +# +# This logic is also abstracted into enable-internal-sources.yml. ConfigFile=$1 CredToken=$2 @@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then ExitWithExitCode 1 fi -if [ -z "$CredToken" ]; then - Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT" - ExitWithExitCode 1 -fi - if [[ `uname -s` == "Darwin" ]]; then NL=$'\\\n' TB='' @@ -105,7 +99,7 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet3.1-internal-transport') fi -DotNetVersions=('5' '6' '7' '8') +DotNetVersions=('5' '6' '7' '8' '9') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; @@ -140,18 +134,20 @@ PackageSources+="$IFS" PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') IFS=$PrevIFS -for FeedName in ${PackageSources[@]} ; do - # Check if there is no existing credential for this FeedName - grep -i "<$FeedName>" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." +if [ "$CredToken" ]; then + for FeedName in ${PackageSources[@]} ; do + # Check if there is no existing credential for this FeedName + grep -i "<$FeedName>" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding credentials for $FeedName." - PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + PackageSourceCredentialsNodeFooter="" + NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" - sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile - fi -done + sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile + fi + done +fi # Re-enable any entries in disabledPackageSources where the feed name contains darc-int grep -i "" $ConfigFile diff --git a/eng/common/build.cmd b/eng/common/build.cmd new file mode 100644 index 000000000000..99daf368abae --- /dev/null +++ b/eng/common/build.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*" +exit /b %ErrorLevel% diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 33a6f2d0e248..438f9920c43e 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -19,6 +19,7 @@ Param( [switch] $pack, [switch] $publish, [switch] $clean, + [switch][Alias('pb')]$productBuild, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -58,6 +59,7 @@ function Print-Usage() { Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" Write-Host " -clean Clean the solution" + Write-Host " -productBuild Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)" Write-Host "" Write-Host "Advanced settings:" @@ -120,6 +122,7 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` + /p:DotNetBuildRepo=$productBuild ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index 50af40cdd2ce..ac1ee8620cd2 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -22,6 +22,9 @@ usage() echo " --sourceBuild Source-build the solution (short: -sb)" echo " Will additionally trigger the following actions: --restore, --build, --pack" echo " If --configuration is not set explicitly, will also set it to 'Release'" + echo " --productBuild Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)" + echo " Will additionally trigger the following actions: --restore, --build, --pack" + echo " If --configuration is not set explicitly, will also set it to 'Release'" echo " --rebuild Rebuild solution" echo " --test Run all unit tests in the solution (short: -t)" echo " --integrationTest Run all integration tests in the solution" @@ -59,6 +62,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" restore=false build=false source_build=false +product_build=false rebuild=false test=false integration_test=false @@ -105,7 +109,7 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; - -excludeCIBinarylog|-nobl) + -excludecibinarylog|-nobl) exclude_ci_binary_log=true ;; -pipelineslog|-pl) @@ -126,6 +130,13 @@ while [[ $# > 0 ]]; do -sourcebuild|-sb) build=true source_build=true + product_build=true + restore=true + pack=true + ;; + -productBuild|-pb) + build=true + product_build=true restore=true pack=true ;; @@ -219,7 +230,9 @@ function Build { /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ + /p:DotNetBuildRepo=$product_build \ /p:ArcadeBuildFromSource=$source_build \ + /p:DotNetBuildSourceOnly=$source_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml new file mode 100644 index 000000000000..8da43d3b5837 --- /dev/null +++ b/eng/common/core-templates/job/job.yml @@ -0,0 +1,253 @@ +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: {} + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + # publishing defaults + artifacts: '' + enableMicrobuild: false + microbuildUseESRP: true + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + componentGovernanceSteps: [] + preSteps: [] + artifactPublishSteps: [] + runAsPublic: false + +# 1es specific parameters + is1ESPipeline: '' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea + ${{ else }}: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} + richNavLogOutputDirectory: $(System.DefaultWorkingDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - ${{ each step in parameters.componentGovernanceSteps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + # Publish test results + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + # gather artifacts + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: CopyFiles@2 + displayName: Gather logs for publish to artifacts + inputs: + SourceFolder: 'artifacts/log' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log' + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - task: CopyFiles@2 + displayName: Gather logs for publish to artifacts + inputs: + SourceFolder: 'artifacts/log/$(_BuildConfig)' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + continueOnError: true + condition: always() + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: CopyFiles@2 + displayName: Gather buildconfiguration for build retry + inputs: + SourceFolder: '$(System.DefaultWorkingDirectory)/eng/common/BuildConfiguration' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration' + continueOnError: true + condition: always() + - ${{ each step in parameters.artifactPublishSteps }}: + - ${{ step }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml new file mode 100644 index 000000000000..edefa789d360 --- /dev/null +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -0,0 +1,121 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(System.DefaultWorkingDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + is1ESPipeline: '' +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Localization Files + pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish LocProject.json + pathToPublish: '$(System.DefaultWorkingDirectory)/eng/Localize/' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml new file mode 100644 index 000000000000..3cb20fb5041f --- /dev/null +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -0,0 +1,172 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + + is1ESPipeline: '' + + repositoryAlias: self + + officialBuildId: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/core-templates/post-build/common-variables.yml + - name: OfficialBuildId + ${{ if ne(parameters.officialBuildId, '') }}: + value: ${{ parameters.officialBuildId }} + ${{ else }}: + value: $(Build.BuildNumber) + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2022.amd64 + os: windows + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - checkout: ${{ parameters.repositoryAlias }} + fetchDepth: 3 + clean: true + + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: AzureCLI@2 + displayName: Publish Build Assets + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(OfficialBuildId) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + $symbolExclusionfile = "$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if (Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs" + } + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish ReleaseConfigs Artifact + pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + publishLocation: Container + artifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + # Darc is targeting 8.0, so make sure it's installed + - task: UseDotNet@2 + inputs: + version: 8.0.x + + - task: AzureCLI@2 + displayName: Publish Using Darc + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: > + -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(System.AccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml new file mode 100644 index 000000000000..d943748ac10e --- /dev/null +++ b/eng/common/core-templates/job/source-build.yml @@ -0,0 +1,99 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # buildArguments: '' + # Specifies additional build arguments to pass to the build script. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + + # Optional list of directories to ignore for component governance scans. + componentGovernanceIgnoreDirectories: [] + + is1ESPipeline: '' + + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/core-templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + ${{ if eq(parameters.is1ESPipeline, 'true') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals build.ubuntu.2004.amd64 + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: build.azurelinux.3.amd64 + os: linux + ${{ else }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + platform: ${{ parameters.platform }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml new file mode 100644 index 000000000000..ddf8c2e00d80 --- /dev/null +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -0,0 +1,81 @@ +parameters: + runAsPublic: false + sourceIndexUploadPackageVersion: 2.0.0-20250425.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + dependsOn: '' + pool: '' + is1ESPipeline: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: 1es-windows-2022-open + os: windows + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET 8 SDK + inputs: + packageType: sdk + version: 8.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: AzureCLI@2 + displayName: Log in to Azure and upload stage1 artifacts to source index + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml new file mode 100644 index 000000000000..4571a7864df6 --- /dev/null +++ b/eng/common/core-templates/jobs/codeql-build.yml @@ -0,0 +1,33 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + is1ESPipeline: '' + +jobs: +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml new file mode 100644 index 000000000000..bf33cdc2cc77 --- /dev/null +++ b/eng/common/core-templates/jobs/jobs.yml @@ -0,0 +1,123 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/core-templates/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + + artifacts: {} + is1ESPipeline: '' + repositoryAlias: self + officialBuildId: '' + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + + - ${{ else }}: + - template: /eng/common/templates/job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + repositoryAlias: ${{ parameters.repositoryAlias }} + officialBuildId: ${{ parameters.officialBuildId }} diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml new file mode 100644 index 000000000000..0b408a67bd51 --- /dev/null +++ b/eng/common/core-templates/jobs/source-build.yml @@ -0,0 +1,63 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/core-templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + + # Optional list of directories to ignore for component governance scans. + componentGovernanceIgnoreDirectories: [] + + is1ESPipeline: '' + + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + enableInternalSources: ${{ parameters.enableInternalSources }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml new file mode 100644 index 000000000000..d5627a994ae5 --- /dev/null +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml new file mode 100644 index 000000000000..864427d9694a --- /dev/null +++ b/eng/common/core-templates/post-build/post-build.yml @@ -0,0 +1,327 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: requireDefaultChannels + displayName: Fail the build if there are no default channel(s) registrations for the current build + type: boolean + default: false + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + + - name: is1ESPipeline + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + os: windows + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 + + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2022.amd64 + os: windows + ${{ else }}: + name: NetCore1ESPool-Publishing-Internal + demands: ImageOverride -equals windows.vs2022.amd64 + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: NuGetAuthenticate@1 + + # Darc is targeting 8.0, so make sure it's installed + - task: UseDotNet@2 + inputs: + version: 8.0.x + + - task: AzureCLI@2 + displayName: Publish Using Darc + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: > + -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(System.AccessToken)' + -WaitPublishingFinish true + -RequireDefaultChannels ${{ parameters.requireDefaultChannels }} + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..a7abd58c4bb6 --- /dev/null +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,74 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + is1ESPipeline: '' + +steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: AzureCLI@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + . $(System.DefaultWorkingDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json + + $BarId = ${{ parameters.BARBuildId }} + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml new file mode 100644 index 000000000000..cf0649aa9565 --- /dev/null +++ b/eng/common/core-templates/steps/component-governance.yml @@ -0,0 +1,16 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + is1ESPipeline: false + displayName: 'Component Detection' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + displayName: ${{ parameters.displayName }} + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/core-templates/steps/enable-internal-runtimes.yml b/eng/common/core-templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..6bdbf62ac500 --- /dev/null +++ b/eng/common/core-templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,32 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true +- name: is1ESPipeline + type: boolean + default: false + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl + is1ESPipeline: ${{ parameters.is1ESPipeline }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml new file mode 100644 index 000000000000..4085512b6909 --- /dev/null +++ b/eng/common/core-templates/steps/enable-internal-sources.yml @@ -0,0 +1,47 @@ +parameters: +# This is the Azure federated service connection that we log into to get an access token. +- name: nugetFederatedServiceConnection + type: string + default: 'dnceng-artifacts-feeds-read' +- name: is1ESPipeline + type: boolean + default: false +# Legacy parameters to allow for PAT usage +- name: legacyCredential + type: string + default: '' + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if ne(parameters.legacyCredential, '') }}: + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token + env: + Token: ${{ parameters.legacyCredential }} + # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. + # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that + # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token. + - ${{ else }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config + - ${{ else }}: + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }} + outputVariableName: 'dnceng-artifacts-feeds-read-access-token' + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + # This is required in certain scenarios to install the ADO credential provider. + # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others + # (e.g. dotnet msbuild). + - task: NuGetAuthenticate@1 diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml new file mode 100644 index 000000000000..7f5b84c4cb82 --- /dev/null +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -0,0 +1,54 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 9.0.0 + BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + is1ESPipeline: false + # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). + publishArtifacts: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- ${{ if eq(parameters.publishArtifacts, 'true')}}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + targetPath: '${{ parameters.manifestDirPath }}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/core-templates/steps/get-delegation-sas.yml b/eng/common/core-templates/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..9db5617ea7de --- /dev/null +++ b/eng/common/core-templates/steps/get-delegation-sas.yml @@ -0,0 +1,55 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' +- name: is1ESPipeline + type: boolean + default: false + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads + # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 + $sas = "" + do { + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + } while($sas.IndexOf('/') -ne -1) + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/core-templates/steps/get-federated-access-token.yml b/eng/common/core-templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..3a4d4410c482 --- /dev/null +++ b/eng/common/core-templates/steps/get-federated-access-token.yml @@ -0,0 +1,42 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: is1ESPipeline + type: boolean +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/core-templates/steps/publish-build-artifacts.yml b/eng/common/core-templates/steps/publish-build-artifacts.yml new file mode 100644 index 000000000000..f24ce346684e --- /dev/null +++ b/eng/common/core-templates/steps/publish-build-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false +- name: args + type: object + default: {} +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml new file mode 100644 index 000000000000..0623ac6e1123 --- /dev/null +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -0,0 +1,58 @@ +parameters: + StageLabel: '' + JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.11' + is1ESPipeline: false + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(System.DefaultWorkingDirectory)/artifacts/log/Debug/* $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + +- task: CopyFiles@2 + displayName: Gather post build logs + inputs: + SourceFolder: '$(System.DefaultWorkingDirectory)/PostBuildLogs' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + +- template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Logs + pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + publishLocation: Container + artifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/core-templates/steps/publish-pipeline-artifacts.yml b/eng/common/core-templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 000000000000..2efec04dc2c1 --- /dev/null +++ b/eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/core-templates/steps/retain-build.yml b/eng/common/core-templates/steps/retain-build.yml new file mode 100644 index 000000000000..83d97a26a01f --- /dev/null +++ b/eng/common/core-templates/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml new file mode 100644 index 000000000000..68fa739c4ab2 --- /dev/null +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -0,0 +1,93 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml new file mode 100644 index 000000000000..7846584d2a77 --- /dev/null +++ b/eng/common/core-templates/steps/source-build.yml @@ -0,0 +1,137 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/core-templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + + # Optional list of directories to ignore for component governance scans. + componentGovernanceIgnoreDirectories: [] + + is1ESPipeline: false + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If file changes are detected, set CopyWipIntoInnerSourceBuildRepo to copy the WIP changes into the inner source build repo. + internalRestoreArgs= + if ! git diff --quiet; then + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + ${{ parameters.platform.buildArguments }} \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:DotNetBuildSourceOnly=true \ + /p:DotNetBuildRepo=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(System.DefaultWorkingDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/sb/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish BuildLogs + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + sbomEnabled: false # we don't need SBOM for logs + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + displayName: Component Detection (Exclude upstream cache) + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ if eq(length(parameters.componentGovernanceIgnoreDirectories), 0) }}: + componentGovernanceIgnoreDirectories: '$(System.DefaultWorkingDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + ${{ else }}: + componentGovernanceIgnoreDirectories: ${{ join(',', parameters.componentGovernanceIgnoreDirectories) }} + disableComponentGovernance: ${{ eq(variables['System.TeamProject'], 'public') }} diff --git a/eng/common/core-templates/variables/pool-providers.yml b/eng/common/core-templates/variables/pool-providers.yml new file mode 100644 index 000000000000..41053d382a2e --- /dev/null +++ b/eng/common/core-templates/variables/pool-providers.yml @@ -0,0 +1,8 @@ +parameters: + is1ESPipeline: false + +variables: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates/variables/pool-providers.yml \ No newline at end of file diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index f163fb9dae96..7e9ba2b75ed3 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -5,15 +5,15 @@ __NDK_Version=r21 usage() { echo "Creates a toolchain and sysroot used for cross-compiling for Android." - echo. + echo echo "Usage: $0 [BuildArch] [ApiLevel]" - echo. + echo echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" - echo. + echo echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" echo "by setting the TOOLCHAIN_DIR environment variable" - echo. + echo echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android." diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 9caf9b021dbd..4b5e8d7166bd 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -8,7 +8,7 @@ usage() echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" - echo " for FreeBSD can be: freebsd12, freebsd13" + echo " for FreeBSD can be: freebsd13, freebsd14" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -30,7 +30,8 @@ __IllumosArch=arm7 __HaikuArch=arm __QEMUArch=arm __UbuntuArch=armhf -__UbuntuRepo="http://ports.ubuntu.com/" +__UbuntuRepo= +__UbuntuSuites="updates security backports" __LLDB_Package="liblldb-3.9-dev" __SkipUnmount=0 @@ -71,9 +72,9 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.4-RELEASE" +__FreeBSDBase="13.3-RELEASE" __FreeBSDPkg="1.17.0" -__FreeBSDABI="12" +__FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" @@ -129,6 +130,7 @@ __AlpineKeys=' 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== ' __Keyring= +__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 __UseMirror=0 @@ -142,7 +144,6 @@ while :; do case $lowerI in -\?|-h|--help) usage - exit 1 ;; arm) __BuildArch=arm @@ -163,6 +164,7 @@ while :; do __UbuntuArch=armel __UbuntuRepo="http://ftp.debian.org/debian/" __CodeName=jessie + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" ;; armv6) __BuildArch=armv6 @@ -170,10 +172,12 @@ while :; do __QEMUArch=arm __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" __CodeName=buster + __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg" __LLDB_Package="liblldb-6.0-dev" + __UbuntuSuites= - if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg" + if [[ -e "$__KeyringFile" ]]; then + __Keyring="--keyring $__KeyringFile" fi ;; riscv64) @@ -182,13 +186,8 @@ while :; do __AlpinePackages="${__AlpinePackages// lldb-dev/}" __QEMUArch=riscv64 __UbuntuArch=riscv64 - __UbuntuRepo="http://deb.debian.org/debian-ports" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" unset __LLDB_Package - - if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring" - fi ;; ppc64le) __BuildArch=ppc64le @@ -229,12 +228,19 @@ while :; do __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; lldb*) - version="${lowerI/lldb/}" - parts=(${version//./ }) + version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" + majorVersion="${version%%.*}" + + [ -z "${version##*.*}" ] && minorVersion="${version#*.}" + if [ -z "$minorVersion" ]; then + minorVersion=0 + fi # for versions > 6.0, lldb has dropped the minor version - if [[ "${parts[0]}" -gt 6 ]]; then - version="${parts[0]}" + if [ "$majorVersion" -le 6 ]; then + version="$majorVersion.$minorVersion" + else + version="$majorVersion" fi __LLDB_Package="liblldb-${version}-dev" @@ -243,15 +249,19 @@ while :; do unset __LLDB_Package ;; llvm*) - version="${lowerI/llvm/}" - parts=(${version//./ }) - __LLVM_MajorVersion="${parts[0]}" - __LLVM_MinorVersion="${parts[1]}" - - # for versions > 6.0, llvm has dropped the minor version - if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then - __LLVM_MinorVersion=0; + version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" + __LLVM_MajorVersion="${version%%.*}" + + [ -z "${version##*.*}" ] && __LLVM_MinorVersion="${version#*.}" + if [ -z "$__LLVM_MinorVersion" ]; then + __LLVM_MinorVersion=0 + fi + + # for versions > 6.0, lldb has dropped the minor version + if [ "$__LLVM_MajorVersion" -gt 6 ]; then + __LLVM_MinorVersion= fi + ;; xenial) # Ubuntu 16.04 if [[ "$__CodeName" != "jessie" ]]; then @@ -278,8 +288,17 @@ while :; do __CodeName=jammy fi ;; + noble) # Ubuntu 24.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=noble + fi + if [[ -n "$__LLDB_Package" ]]; then + __LLDB_Package="liblldb-18-dev" + fi + ;; jessie) # Debian 8 __CodeName=jessie + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -288,6 +307,7 @@ while :; do stretch) # Debian 9 __CodeName=stretch __LLDB_Package="liblldb-6.0-dev" + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -296,6 +316,7 @@ while :; do buster) # Debian 10 __CodeName=buster __LLDB_Package="liblldb-6.0-dev" + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -303,6 +324,15 @@ while :; do ;; bullseye) # Debian 11 __CodeName=bullseye + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi + ;; + bookworm) # Debian 12 + __CodeName=bookworm + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -310,6 +340,7 @@ while :; do ;; sid) # Debian sid __CodeName=sid + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -323,25 +354,24 @@ while :; do alpine*) __CodeName=alpine __UbuntuRepo= - version="${lowerI/alpine/}" - if [[ "$version" == "edge" ]]; then + if [[ "$lowerI" == "alpineedge" ]]; then __AlpineVersion=edge else - parts=(${version//./ }) - __AlpineMajorVersion="${parts[0]}" - __AlpineMinoVersion="${parts[1]}" - __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinoVersion" + version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" + __AlpineMajorVersion="${version%%.*}" + __AlpineMinorVersion="${version#*.}" + __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinorVersion" fi ;; - freebsd12) + freebsd13) __CodeName=freebsd __SkipUnmount=1 ;; - freebsd13) + freebsd14) __CodeName=freebsd - __FreeBSDBase="13.2-RELEASE" - __FreeBSDABI="13" + __FreeBSDBase="14.0-RELEASE" + __FreeBSDABI="14" __SkipUnmount=1 ;; illumos) @@ -420,6 +450,10 @@ fi __UbuntuPackages+=" ${__LLDB_Package:-}" +if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ports.ubuntu.com/" +fi + if [[ -n "$__LLVM_MajorVersion" ]]; then __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" fi @@ -442,13 +476,39 @@ fi mkdir -p "$__RootfsDir" __RootfsDir="$( cd "$__RootfsDir" && pwd )" +__hasWget= +ensureDownloadTool() +{ + if command -v wget &> /dev/null; then + __hasWget=1 + elif command -v curl &> /dev/null; then + __hasWget=0 + else + >&2 echo "ERROR: either wget or curl is required by this script." + exit 1 + fi +} + if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.12.11 - __ApkToolsSHA512SUM=53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33 __ApkToolsDir="$(mktemp -d)" __ApkKeysDir="$(mktemp -d)" + arch="$(uname -m)" - wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//v$__ApkToolsVersion/x86_64/apk.static" -P "$__ApkToolsDir" + ensureDownloadTool + + if [[ "$__hasWget" == 1 ]]; then + wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + else + curl -SLO --create-dirs --output-dir "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + fi + if [[ "$arch" == "x86_64" ]]; then + __ApkToolsSHA512SUM="53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33" + elif [[ "$arch" == "aarch64" ]]; then + __ApkToolsSHA512SUM="9e2b37ecb2b56c05dad23d379be84fd494c14bd730b620d0d576bda760588e1f2f59a7fcb2f2080577e0085f23a0ca8eadd993b4e61c2ab29549fdb71969afd0" + else + echo "WARNING: add missing hash for your host architecture. To find the value, use: 'find /tmp -name apk.static -exec sha512sum {} \;'" + fi echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c chmod +x "$__ApkToolsDir/apk.static" @@ -477,20 +537,23 @@ if [[ "$__CodeName" == "alpine" ]]; then fi # initialize DB + # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then + # shellcheck disable=SC2086 __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ - search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')" + search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')" fi # install all packages in one go + # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ @@ -501,12 +564,23 @@ if [[ "$__CodeName" == "alpine" ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p "$__RootfsDir"/usr/local/etc JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + + ensureDownloadTool + + if [[ "$__hasWget" == 1 ]]; then + wget -O- "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + else + curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + fi echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf mkdir -p "$__RootfsDir"/tmp # get and build package manager - wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + else + curl -SL "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + fi cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # needed for install to succeed mkdir -p "$__RootfsDir"/host/etc @@ -514,27 +588,43 @@ elif [[ "$__CodeName" == "freebsd" ]]; then rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # install packages we need. INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update + # shellcheck disable=SC2086 INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} + + ensureDownloadTool + echo "Downloading sysroot." - wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + else + curl -SL https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + fi echo "Building binutils. Please wait.." - wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf - + else + curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf - + fi mkdir build-binutils && cd build-binutils - ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" + ../binutils-2.42/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" make -j "$JOBS" && make install && cd .. echo "Building gcc. Please wait.." - wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf - + else + curl -SL https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf - + fi CFLAGS="-fPIC" CXXFLAGS="-fPIC" CXXFLAGS_FOR_TARGET="-fPIC" CFLAGS_FOR_TARGET="-fPIC" export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET mkdir build-gcc && cd build-gcc - ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + ../gcc-13.3.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --disable-libquadmath-support --disable-shared --enable-tls make -j "$JOBS" && make install && cd .. @@ -542,9 +632,13 @@ elif [[ "$__CodeName" == "illumos" ]]; then if [[ "$__UseMirror" == 1 ]]; then BaseUrl=https://pkgsrc.smartos.skylime.net fi - BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" + BaseUrl="$BaseUrl/packages/SmartOS/2019Q4/${__illumosArch}/All" echo "Downloading manifest" - wget "$BaseUrl" + if [[ "$__hasWget" == 1 ]]; then + wget "$BaseUrl" + else + curl -SLO "$BaseUrl" + fi echo "Downloading dependencies." read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do @@ -552,7 +646,11 @@ elif [[ "$__CodeName" == "illumos" ]]; then # find last occurrence of package in listing and extract its name package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)" echo "Resolved name '$package'" - wget "$BaseUrl"/"$package".tgz + if [[ "$__hasWget" == 1 ]]; then + wget "$BaseUrl"/"$package".tgz + else + curl -SLO "$BaseUrl"/"$package".tgz + fi ar -x "$package".tgz tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null done @@ -561,10 +659,17 @@ elif [[ "$__CodeName" == "illumos" ]]; then rm -rf "$__RootfsDir"/{tmp,+*} mkdir -p "$__RootfsDir"/usr/include/net mkdir -p "$__RootfsDir"/usr/include/netpacket - wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h - wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h - wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h - wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h + if [[ "$__hasWget" == 1 ]]; then + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h + else + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h + fi elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} @@ -574,9 +679,16 @@ elif [[ "$__CodeName" == "haiku" ]]; then mkdir "$__RootfsDir/tmp/download" + ensureDownloadTool + echo "Downloading Haiku package tool" - git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script - wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) + git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script" + if [[ "$__hasWget" == 1 ]]; then + wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" + else + curl -SLo "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" + fi + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" @@ -589,14 +701,25 @@ elif [[ "$__CodeName" == "haiku" ]]; then echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + if [[ "$__hasWget" == 1 ]]; then + hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + else + hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + fi done for package in haiku haiku_devel; do echo "Downloading $package..." - hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + if [[ "$__hasWget" == 1 ]]; then + hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + else + hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + fi done # Set up the sysroot @@ -609,7 +732,11 @@ elif [[ "$__CodeName" == "haiku" ]]; then # Download buildtools echo "Downloading Haiku buildtools" - wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) + if [[ "$__hasWget" == 1 ]]; then + wget -O "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)" + else + curl -SLo "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)" + fi unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" # Cleaning up temporary files @@ -622,10 +749,22 @@ elif [[ -n "$__CodeName" ]]; then __Keyring="$__Keyring --force-check-gpg" fi + # shellcheck disable=SC2086 + echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" - cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list" + + mkdir -p "$__RootfsDir/etc/apt/sources.list.d/" + cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" <>Start configuring Tizen rootfs" ln -sfn asm-${LINK_ARCH} ./usr/include/asm patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +if [[ "$TIZEN_ARCH" == "riscv64" ]]; then + echo "Fixing broken symlinks in $PWD" + rm ./usr/lib64/libresolv.so + ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so + rm ./usr/lib64/libpthread.so + ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so + rm ./usr/lib64/libdl.so + ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so + rm ./usr/lib64/libutil.so + ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so + rm ./usr/lib64/libm.so + ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so + rm ./usr/lib64/librt.so + ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so + rm ./lib/ld-linux-riscv64-lp64d.so.1 + ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1 +fi echo "<:--stdlib=${CLR_CMAKE_CXX_STANDARD_LIBRARY}>) + add_link_options($<$:--stdlib=${CLR_CMAKE_CXX_STANDARD_LIBRARY}>) +endif() + +option(CLR_CMAKE_CXX_STANDARD_LIBRARY_STATIC "Statically link against the C++ standard library" OFF) +if(CLR_CMAKE_CXX_STANDARD_LIBRARY_STATIC) + add_link_options($<$:-static-libstdc++>) +endif() + +set(CLR_CMAKE_CXX_ABI_LIBRARY "" CACHE STRING "C++ ABI implementation library to link against. Only supported with the Clang compiler.") +if (CLR_CMAKE_CXX_ABI_LIBRARY) + # The user may specify the ABI library with the 'lib' prefix, like 'libstdc++'. Strip the prefix here so the linker finds the right library. + string(REGEX REPLACE "^lib(.+)" "\\1" CLR_CMAKE_CXX_ABI_LIBRARY ${CLR_CMAKE_CXX_ABI_LIBRARY}) + # We need to specify this as a linker-backend option as Clang will filter this option out when linking to libc++. + add_link_options("LINKER:-l${CLR_CMAKE_CXX_ABI_LIBRARY}") +endif() + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 8fda30bdce2b..e33743105635 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,6 +1,6 @@ param ( $darcVersion = $null, - $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16', + $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20', $verbosity = 'minimal', $toolpath = $null ) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index c305ae6bd771..36dbd45e1ce8 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -2,7 +2,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' -versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16' +versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20' verbosity='minimal' while [[ $# > 0 ]]; do diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 7e69e3a9e24a..7b9d97e3bd4d 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -71,6 +71,9 @@ case $cpuname in i[3-6]86) buildarch=x86 ;; + riscv64) + buildarch=riscv64 + ;; *) echo "Unknown CPU $cpuname detected, treating it as x64" buildarch=x64 @@ -82,7 +85,7 @@ if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi -InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { +InstallDotNet "$dotnetRoot" $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { local exit_code=$? Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh index bbb4922151e6..b8ecca72bbf5 100644 --- a/eng/common/generate-sbom-prep.sh +++ b/eng/common/generate-sbom-prep.sh @@ -14,10 +14,10 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" . $scriptroot/pipeline-logging-functions.sh + # replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" - manifest_dir=$1 # Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj index d7f185856e79..c1323bf41210 100644 --- a/eng/common/helixpublish.proj +++ b/eng/common/helixpublish.proj @@ -1,3 +1,4 @@ + diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index dbf99d82a5c2..f1d041c33da5 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,4 +1,11 @@ + + + false + false + + + diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config index 19d3d311b166..f70261ed689b 100644 --- a/eng/common/internal/NuGet.config +++ b/eng/common/internal/NuGet.config @@ -4,4 +4,7 @@ + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 7f5ce6d60813..feaa6d20812d 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,9 +1,10 @@ + net472 - false false + false @@ -14,17 +15,8 @@ - - - - https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; - - - $(RestoreSources); - https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; - - + diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index ca38268c44d8..f71f6af6cdbc 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -277,7 +277,8 @@ function Get-MachineArchitecture { if (($ProcessorArchitecture -Eq "AMD64") -Or ($ProcessorArchitecture -Eq "IA64") -Or ($ProcessorArchitecture -Eq "ARM64") -Or - ($ProcessorArchitecture -Eq "LOONGARCH64")) { + ($ProcessorArchitecture -Eq "LOONGARCH64") -Or + ($ProcessorArchitecture -Eq "RISCV64")) { return "x64" } return "x86" diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 2d5660642b8d..9a0e1f2b4567 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -2,7 +2,9 @@ # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # -# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! +# NOTE: some scripts source this file and rely on stdout being empty, make sure +# to not output *anything* here, unless it is an error message that fails the +# build. if [ -z "$build_arch" ] || [ -z "$compiler" ]; then echo "Usage..." @@ -17,11 +19,9 @@ case "$compiler" in # clangx.y or clang-x.y version="$(echo "$compiler" | tr -d '[:alpha:]-=')" majorVersion="${version%%.*}" - [ -z "${version##*.*}" ] && minorVersion="${version#*.}" - if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then - minorVersion=0; - fi + # LLVM based on v18 released in early 2024, with two releases per year + maxVersion="$((18 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 6)))" compiler=clang ;; @@ -29,7 +29,9 @@ case "$compiler" in # gccx.y or gcc-x.y version="$(echo "$compiler" | tr -d '[:alpha:]-=')" majorVersion="${version%%.*}" - [ -z "${version##*.*}" ] && minorVersion="${version#*.}" + + # GCC based on v14 released in early 2024, with one release per year + maxVersion="$((14 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 12)))" compiler=gcc ;; esac @@ -47,91 +49,98 @@ check_version_exists() { desired_version=-1 # Set up the environment to be used for building with the desired compiler. - if command -v "$compiler-$1.$2" > /dev/null; then - desired_version="-$1.$2" - elif command -v "$compiler$1$2" > /dev/null; then - desired_version="$1$2" - elif command -v "$compiler-$1$2" > /dev/null; then - desired_version="-$1$2" + if command -v "$compiler-$1" > /dev/null; then + desired_version="-$1" + elif command -v "$compiler$1" > /dev/null; then + desired_version="$1" fi echo "$desired_version" } +__baseOS="$(uname)" +set_compiler_version_from_CC() { + if [ "$__baseOS" = "Darwin" ]; then + # On Darwin, the versions from -version/-dumpversion refer to Xcode + # versions, not llvm versions, so we can't rely on them. + return + fi + + version="$("$CC" -dumpversion)" + if [ -z "$version" ]; then + echo "Error: $CC -dumpversion didn't provide a version" + exit 1 + fi + + # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments. + IFS=. read -r majorVersion _ < /dev/null; then - if [ "$(uname)" != "Darwin" ]; then - echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH." - fi - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "No usable version of $compiler found." + if ! command -v "$compiler" > /dev/null; then + echo "Error: No compatible version of $compiler was found within the range of $minVersion to $maxVersion. Please upgrade your toolchain or specify the compiler explicitly using CLR_CC and CLR_CXX environment variables." exit 1 fi - else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then - if command -v "$compiler" > /dev/null; then - echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." - exit 1 - fi - fi - fi + + CC="$(command -v "$compiler" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler" 2> /dev/null)" + set_compiler_version_from_CC fi else - desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" + desired_version="$(check_version_exists "$majorVersion")" if [ "$desired_version" = "-1" ]; then - echo "Could not find specific version of $compiler: $majorVersion $minorVersion." + echo "Error: Could not find specific version of $compiler: $majorVersion." exit 1 fi fi if [ -z "$CC" ]; then - CC="$(command -v "$compiler$desired_version")" - CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi + CC="$(command -v "$compiler$desired_version" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)" + if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi + set_compiler_version_from_CC fi else if [ ! -f "$CLR_CC" ]; then - echo "CLR_CC is set but path '$CLR_CC' does not exist" + echo "Error: CLR_CC is set but path '$CLR_CC' does not exist" exit 1 fi CC="$CLR_CC" CXX="$CLR_CXX" + set_compiler_version_from_CC fi if [ -z "$CC" ]; then - echo "Unable to find $compiler." + echo "Error: Unable to find $compiler." exit 1 fi -# Only lld version >= 9 can be considered stable. lld doesn't support s390x. -if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && [ "$build_arch" != "s390x" ]; then - if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then - LDFLAGS="-fuse-ld=lld" +if [ "$__baseOS" != "Darwin" ]; then + # On Darwin, we always want to use the Apple linker. + + # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. + if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then + if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then + LDFLAGS="-fuse-ld=lld" + fi fi fi -SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" +SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)" export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index de1687b2ccbe..83ea7aab0e08 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # getNonPortableDistroRid # @@ -11,21 +11,16 @@ # non-portable rid getNonPortableDistroRid() { - local targetOs="$1" - local targetArch="$2" - local rootfsDir="$3" - local nonPortableRid="" + targetOs="$1" + targetArch="$2" + rootfsDir="$3" + nonPortableRid="" if [ "$targetOs" = "linux" ]; then + # shellcheck disable=SC1091 if [ -e "${rootfsDir}/etc/os-release" ]; then - source "${rootfsDir}/etc/os-release" - - if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then - # remove the last version digit - VERSION_ID="${VERSION_ID%.*}" - fi - - if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then + . "${rootfsDir}/etc/os-release" + if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" else # Rolling release distros either do not set VERSION_ID, set it as blank or @@ -33,45 +28,33 @@ getNonPortableDistroRid() # so omit it here to be consistent with everything else. nonPortableRid="${ID}-${targetArch}" fi - elif [ -e "${rootfsDir}/android_platform" ]; then - source "$rootfsDir"/android_platform + # shellcheck disable=SC1091 + . "${rootfsDir}/android_platform" nonPortableRid="$RID" fi fi if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is shell script and it should always work. - __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; }) + # $rootfsDir can be empty. freebsd-version is a shell script and should always work. + __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then + elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" elif [ "$targetOs" = "illumos" ]; then __uname_version=$(uname -v) - case "$__uname_version" in - omnios-*) - __omnios_major_version=$(echo "${__uname_version:8:2}") - nonPortableRid=omnios."$__omnios_major_version"-"$targetArch" - ;; - joyent_*) - __smartos_major_version=$(echo "${__uname_version:7:4}") - nonPortableRid=smartos."$__smartos_major_version"-"$targetArch" - ;; - illumos_*) - nonPortableRid=openindiana-"$targetArch" - ;; - esac + nonPortableRid="illumos-${targetArch}" elif [ "$targetOs" = "solaris" ]; then __uname_version=$(uname -v) - __solaris_major_version=$(echo "${__uname_version%.*}") - nonPortableRid=solaris."$__solaris_major_version"-"$targetArch" + __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) + nonPortableRid="solaris.$__solaris_major_version-${targetArch}" elif [ "$targetOs" = "haiku" ]; then - __uname_release=$(uname -r) + __uname_release="$(uname -r)" nonPortableRid=haiku.r"$__uname_release"-"$targetArch" fi - echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')" + echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' } # initDistroRidGlobal @@ -85,26 +68,23 @@ getNonPortableDistroRid() # None # # Notes: -# -# It is important to note that the function does not return anything, but it -# exports the following variables on success: -# -# __DistroRid : Non-portable rid of the target platform. -# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. -# +# It is important to note that the function does not return anything, but it +# exports the following variables on success: +# __DistroRid : Non-portable rid of the target platform. +# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. initDistroRidGlobal() { - local targetOs="$1" - local targetArch="$2" - local rootfsDir="" - if [ "$#" -ge 3 ]; then + targetOs="$1" + targetArch="$2" + rootfsDir="" + if [ $# -ge 3 ]; then rootfsDir="$3" fi if [ -n "${rootfsDir}" ]; then # We may have a cross build. Check for the existence of the rootfsDir if [ ! -e "${rootfsDir}" ]; then - echo "Error rootfsDir has been passed, but the location is not valid." + echo "Error: rootfsDir has been passed, but the location is not valid." exit 1 fi fi @@ -119,7 +99,7 @@ initDistroRidGlobal() STRINGS="$(command -v llvm-strings || true)" fi - # Check for musl-based distros (e.g Alpine Linux, Void Linux). + # Check for musl-based distros (e.g. Alpine Linux, Void Linux). if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then __PortableTargetOS="linux-musl" diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh index e693617a6c2b..38921d4338f7 100644 --- a/eng/common/native/init-os-and-arch.sh +++ b/eng/common/native/init-os-and-arch.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # Use uname to determine what the OS is. OSName=$(uname -s | tr '[:upper:]' '[:lower:]') @@ -35,6 +35,10 @@ fi case "$CPUName" in arm64|aarch64) arch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + arch=arm + fi ;; loongarch64) @@ -50,6 +54,7 @@ case "$CPUName" in ;; armv7l|armv8l) + # shellcheck disable=SC1091 if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then arch=armel else diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 index 63f3464c986a..61208d2d1351 100644 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -4,10 +4,18 @@ param( ) try { - . $PSScriptRoot\post-build-utils.ps1 + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + + # `tools.ps1` checks $ci to perform some actions. Since the post-build + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + $disableConfigureToolsetImport = $true + . $PSScriptRoot\..\tools.ps1 if ($PromoteToChannels -eq "") { - Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." ExitWithExitCode 0 } diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 index 22b1c4dfe4a7..e5de00c89836 100644 --- a/eng/common/post-build/nuget-validation.ps1 +++ b/eng/common/post-build/nuget-validation.ps1 @@ -5,9 +5,14 @@ param( [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are ) -try { - . $PSScriptRoot\post-build-utils.ps1 +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +$disableConfigureToolsetImport = $true +. $PSScriptRoot\..\tools.ps1 +try { & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg } catch { diff --git a/eng/common/post-build/nuget-verification.ps1 b/eng/common/post-build/nuget-verification.ps1 index 6cbbcafade26..eea88e653c91 100644 --- a/eng/common/post-build/nuget-verification.ps1 +++ b/eng/common/post-build/nuget-verification.ps1 @@ -118,4 +118,4 @@ if ($LASTEXITCODE -ne 0) { Write-Error "The verify tool found some problems." } else { Write-Output "The verify tool succeeded." -} \ No newline at end of file +} diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 238945cb5ab4..a261517ef906 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -5,11 +5,17 @@ param( [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, - [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters + [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters, + [Parameter(Mandatory=$false)][string] $RequireDefaultChannels ) try { - . $PSScriptRoot\post-build-utils.ps1 + # `tools.ps1` checks $ci to perform some actions. Since the post-build + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + $disableConfigureToolsetImport = $true + . $PSScriptRoot\..\tools.ps1 $darc = Get-Darc @@ -28,6 +34,10 @@ try { if ("false" -eq $WaitPublishingFinish) { $optionalParams.Add("--no-wait") | Out-Null } + + if ("true" -eq $RequireDefaultChannels) { + $optionalParams.Add("--default-channels-required") | Out-Null + } & $darc add-build-to-channel ` --id $buildId ` @@ -37,6 +47,7 @@ try { --azdev-pat "$AzdoToken" ` --bar-uri "$MaestroApiEndPoint" ` --ci ` + --verbose ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 new file mode 100644 index 000000000000..b7fc19591507 --- /dev/null +++ b/eng/common/post-build/redact-logs.ps1 @@ -0,0 +1,89 @@ +[CmdletBinding(PositionalBinding=$False)] +param( + [Parameter(Mandatory=$true, Position=0)][string] $InputPath, + [Parameter(Mandatory=$true)][string] $BinlogToolVersion, + [Parameter(Mandatory=$false)][string] $DotnetPath, + [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json', + # File with strings to redact - separated by newlines. + # For comments start the line with '# ' - such lines are ignored + [Parameter(Mandatory=$false)][string] $TokensFilePath, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact +) + +try { + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + + # `tools.ps1` checks $ci to perform some actions. Since the post-build + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + $disableConfigureToolsetImport = $true + . $PSScriptRoot\..\tools.ps1 + + $packageName = 'binlogtool' + + $dotnet = $DotnetPath + + if (!$dotnet) { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + } + + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$packageName*") { + & "$dotnet" tool uninstall $packageName -g + } + + $toolPath = "$PSScriptRoot\..\..\..\.tools" + $verbosity = 'minimal' + + New-Item -ItemType Directory -Force -Path $toolPath + + Push-Location -Path $toolPath + + try { + Write-Host "Installing Binlog redactor CLI..." + Write-Host "'$dotnet' new tool-manifest" + & "$dotnet" new tool-manifest + Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + + if (Test-Path $TokensFilePath) { + Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath + $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " } + } + + $optionalParams = [System.Collections.ArrayList]::new() + + Foreach ($p in $TokensToRedact) + { + if($p -match '^\$\(.*\)$') + { + Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable" -f $p) + } + elseif($p) + { + $optionalParams.Add("-p:" + $p) | Out-Null + } + } + + & $dotnet binlogtool redact --input:$InputPath --recurse --in-place ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now." + } + } + finally { + Pop-Location + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_" + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 4011d324e739..1976ef70fb85 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -6,7 +6,15 @@ param( [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use ) -. $PSScriptRoot\post-build-utils.ps1 +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +$disableConfigureToolsetImport = $true +. $PSScriptRoot\..\tools.ps1 # Cache/HashMap (File -> Exist flag) used to consult whether a file exist # in the repository at a specific commit point. This is populated by inserting diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index cd2181bafa05..7146e593ffae 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -322,8 +322,6 @@ function InstallDotnetSymbol { } try { - . $PSScriptRoot\post-build-utils.ps1 - InstallDotnetSymbol foreach ($Job in @(Get-Job)) { diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 5bfbb02ef043..3849bdb3cf51 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -5,11 +5,11 @@ - + - + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 81ded5b7f477..4715d75e974d 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -6,6 +6,7 @@ Param( [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list # format. @@ -74,7 +75,7 @@ try { } Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel } $gdnFolder = Join-Path $workingDirectory '.gdn' @@ -103,6 +104,7 @@ try { -TargetDirectory $targetDirectory ` -GdnFolder $gdnFolder ` -ToolsList $tools ` + -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` -GuardianLoggerLevel $GuardianLoggerLevel ` -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 588ff8e22fbe..3ac1d92b3700 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -3,6 +3,7 @@ Param( [string] $Repository, [string] $BranchName='master', [string] $WorkingDirectory, + [string] $AzureDevOpsAccessToken, [string] $GuardianLoggerLevel='Standard' ) @@ -20,7 +21,14 @@ $ci = $true # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" +$zipFile = "$WorkingDirectory/gdn.zip" + Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory '.gdn') try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index e5f543ea68c2..4585cfd6bba1 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 7fe603fe995d..648c5068d7d6 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -4,8 +4,6 @@ function Install-Gdn { [Parameter(Mandatory=$true)] [string]$Path, - [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json", - # If omitted, install the latest version of Guardian, otherwise install that specific version. [string]$Version ) @@ -21,7 +19,7 @@ function Install-Gdn { $ci = $true . $PSScriptRoot\..\tools.ps1 - $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") if ($Version) { $argumentList += "-Version $Version" diff --git a/eng/common/sdl/trim-assets-version.ps1 b/eng/common/sdl/trim-assets-version.ps1 index a2e004877045..0daa2a9e9462 100644 --- a/eng/common/sdl/trim-assets-version.ps1 +++ b/eng/common/sdl/trim-assets-version.ps1 @@ -72,4 +72,4 @@ catch { Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md new file mode 100644 index 000000000000..4bf4cf41bd7c --- /dev/null +++ b/eng/common/template-guidance.md @@ -0,0 +1,133 @@ +# Overview + +Arcade provides templates for public (`/templates`) and 1ES pipeline templates (`/templates-official`) scenarios. Pipelines which are required to be managed by 1ES pipeline templates should reference `/templates-offical`, all other pipelines may reference `/templates`. + +## How to use + +Basic guidance is: + +- 1ES Pipeline Template or 1ES Microbuild template runs should reference `eng/common/templates-official`. Any internal production-graded pipeline should use these templates. + +- All other runs should reference `eng/common/templates`. + +See [azure-pipelines.yml](../../azure-pipelines.yml) (templates-official example) or [azure-pipelines-pr.yml](../../azure-pipelines-pr.yml) (templates example) for examples. + +#### The `templateIs1ESManaged` parameter + +The `templateIs1ESManaged` is available on most templates and affects which of the variants is used for nested templates. See [Development Notes](#development-notes) below for more information on the `templateIs1ESManaged1 parameter. + +- For templates under `job/`, `jobs/`, `steps`, or `post-build/`, this parameter must be explicitly set. + +## Multiple outputs + +1ES pipeline templates impose a policy where every publish artifact execution results in additional security scans being injected into your pipeline. When using `templates-official/jobs/jobs.yml`, Arcade reduces the number of additional security injections by gathering all publishing outputs into the [Build.ArtifactStagingDirectory](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services), and utilizing the [outputParentDirectory](https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs) feature of 1ES pipeline templates. When implementing your pipeline, if you ensure publish artifacts are located in the `$(Build.ArtifactStagingDirectory)`, and utilize the 1ES provided template context, then you can reduce the number of security scans for your pipeline. + +Example: +``` yaml +# azure-pipelines.yml +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + stages: + - stage: build + jobs: + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + # 1ES makes use of outputs to reduce security task injection overhead + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish logs from source' + continueOnError: true + condition: always() + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/log + artifactName: Logs + jobs: + - job: Windows + steps: + - script: echo "friendly neighborhood" > artifacts/marvel/spiderman.txt + # copy build outputs to artifact staging directory for publishing + - task: CopyFiles@2 + displayName: Gather build output + inputs: + SourceFolder: '$(System.DefaultWorkingDirectory)/artifacts/marvel' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel' +``` + +Note: Multiple outputs are ONLY applicable to 1ES PT publishing (only usable when referencing `templates-official`). + +## Development notes + +**Folder / file structure** + +``` text +eng\common\ + [templates || templates-official]\ + job\ + job.yml (shim + artifact publishing logic) + onelocbuild.yml (shim) + publish-build-assets.yml (shim) + source-build.yml (shim) + source-index-stage1.yml (shim) + jobs\ + codeql-build.yml (shim) + jobs.yml (shim) + source-build.yml (shim) + post-build\ + post-build.yml (shim) + common-variabls.yml (shim) + setup-maestro-vars.yml (shim) + steps\ + publish-build-artifacts.yml (logic) + publish-pipeline-artifacts.yml (logic) + component-governance.yml (shim) + generate-sbom.yml (shim) + publish-logs.yml (shim) + retain-build.yml (shim) + send-to-helix.yml (shim) + source-build.yml (shim) + variables\ + pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project + sdl-variables.yml (logic) + core-templates\ + job\ + job.yml (logic) + onelocbuild.yml (logic) + publish-build-assets.yml (logic) + source-build.yml (logic) + source-index-stage1.yml (logic) + jobs\ + codeql-build.yml (logic) + jobs.yml (logic) + source-build.yml (logic) + post-build\ + common-variabls.yml (logic) + post-build.yml (logic) + setup-maestro-vars.yml (logic) + steps\ + component-governance.yml (logic) + generate-sbom.yml (logic) + publish-build-artifacts.yml (redirect) + publish-logs.yml (logic) + publish-pipeline-artifacts.yml (redirect) + retain-build.yml (logic) + send-to-helix.yml (logic) + source-build.yml (logic) + variables\ + pool-providers.yml (redirect) +``` + +In the table above, a file is designated as "shim", "logic", or "redirect". + +- shim - represents a yaml file which is an intermediate step between pipeline logic and .Net Core Engineering's templates (`core-templates`) and defines the `is1ESPipeline` parameter value. + +- logic - represents actual base template logic. + +- redirect- represents a file in `core-templates` which redirects to the "logic" file in either `templates` or `templates-official`. + +Logic for Arcade's templates live **primarily** in the `core-templates` folder. The exceptions to the location of the logic files are around artifact publishing, which is handled differently between 1es pipeline templates and standard templates. `templates` and `templates-official` provide shim entry points which redirect to `core-templates` while also defining the `is1ESPipeline` parameter. If a shim is referenced in `templates`, then `is1ESPipeline` is set to `false`. If a shim is referenced in `templates-official`, then `is1ESPipeline` is set to `true`. + +Within `templates` and `templates-official`, the templates at the "stages", and "jobs" / "job" level have been replaced with shims. Templates at the "steps" and "variables" level are typically too granular to be replaced with shims and instead persist logic which is directly applicable to either scenario. + +Within `core-templates`, there are a handful of places where logic is dependent on which shim entry point was used. In those places, we redirect back to the respective logic file in `templates` or `templates-official`. diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 4cca1114fcca..81ea7a261f2d 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,271 +1,81 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - microbuildUseESRP: true - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false # Sbom related params enableSbom: true - PackageVersion: 7.0.0 + runAsPublic: false + PackageVersion: 9.0.0 BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(parameters.microbuildUseESRP, true) }}: - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea - ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} - richNavLogOutputDirectory: $(System.DefaultWorkingDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates-official/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: 'artifacts/log' - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: 'Publish logs' - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates-official/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: '$(System.DefaultWorkingDirectory)\eng\common\BuildConfiguration' - artifactName: 'BuildConfiguration' - displayName: 'Publish build retry configuration' - continueOnError: true +- template: /eng/common/core-templates/job/job.yml + parameters: + is1ESPipeline: true + + componentGovernanceSteps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion }} + BuildDropPath: ${{ parameters.buildDropPath }} + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + publishArtifacts: false + + # publish artifacts + # for 1ES managed templates, use the templateContext.output to handle multiple outputs. + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - output: buildArtifacts + displayName: Publish pipeline artifacts + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: always() + continueOnError: true + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + sbomEnabled: false # we don't need SBOM for logs + + - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: + - output: buildArtifacts + displayName: Publish Logs + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + sbomEnabled: false # we don't need SBOM for logs + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/eng/common/BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true + sbomEnabled: false # we don't need SBOM for BuildConfiguration + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - output: pipelineArtifact + displayName: Publish SBOM manifest + continueOnError: true + targetPath: $(Build.ArtifactStagingDirectory)/sbom + artifactName: $(ARTIFACT_NAME) + + # add any outputs provided via root yaml + - ${{ if ne(parameters.templateContext.outputs, '') }}: + - ${{ each output in parameters.templateContext.outputs }}: + - ${{ output }} + + # add any remaining templateContext properties + ${{ each context in parameters.templateContext }}: + ${{ if and(ne(context.key, 'outputParentDirectory'), ne(context.key, 'outputs')) }}: + ${{ context.key }}: ${{ context.value }} + + ${{ each parameter in parameters }}: + ${{ if and(ne(parameter.key, 'templateContext'), ne(parameter.key, 'is1ESPipeline')) }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index 68e7a65605c5..0f0c514b912d 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -1,112 +1,7 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(System.DefaultWorkingDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: true - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 67a0de433a3d..d667a70e8de7 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -1,177 +1,7 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - - repositoryAlias: self - - officialBuildId: '' - jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/common-variables.yml - - name: OfficialBuildId - ${{ if ne(parameters.officialBuildId, '') }}: - value: ${{ parameters.officialBuildId }} - ${{ else }}: - value: $(Build.BuildNumber) - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2022.amd64 - os: windows - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: ${{ parameters.repositoryAlias }} - fetchDepth: 3 - clean: true - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Build Assets - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1 - arguments: > - -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(OfficialBuildId) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force - $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - Add-Content -Path $filePath -Value $(BARBuildId) - Add-Content -Path $filePath -Value "$(DefaultChannels)" - Add-Content -Path $filePath -Value $(IsStableBuild) - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: true - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates-official/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 5b1182529034..1a480034b678 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -1,79 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - - # Optional list of directories to ignore for component governance scans. - cgIgnoreDirectories: [] - - # If set to true and running on a non-public project, - # Internal blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: build.azurelinux.3.amd64 - os: linux - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: true - steps: - - ${{ if eq(parameters.enableInternalSources, true) }}: - - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - - template: /eng/common/templates-official/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} - cgIgnoreDirectories: ${{ parameters.cgIgnoreDirectories }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 2f1304ede988..6d5ead316f92 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,83 +1,7 @@ -parameters: - runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') - dependsOn: '' - pool: '' - jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexUploadPackageVersion - value: ${{ parameters.sourceIndexUploadPackageVersion }} - - name: SourceIndexProcessBinlogPackageVersion - value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - os: windows - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - displayName: Get stage 1 auth token - inputs: - azureSubscription: 'SourceDotNet Stage1 Publish' - addSpnToEnvironment: true - scriptType: 'ps' - scriptLocation: 'inlineScript' - inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - - - script: | - az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) - displayName: "Login to Azure" +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: true - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml index f6476912a861..a726322ecfe0 100644 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -1,31 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - jobs: -- template: /eng/common/templates-official/jobs/jobs.yml +- template: /eng/common/core-templates/jobs/codeql-build.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true + is1ESPipeline: true - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml index 03aa64e1741f..007deddaea0f 100644 --- a/eng/common/templates-official/jobs/jobs.yml +++ b/eng/common/templates-official/jobs/jobs.yml @@ -1,101 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates-official/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - repositoryAlias: self - officialBuildId: '' - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates-official/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: true - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} - repositoryAlias: ${{ parameters.repositoryAlias }} - officialBuildId: ${{ parameters.officialBuildId }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 21a346fbd6c7..483e7b611f34 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -1,59 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates-official/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - - # Optional list of directories to ignore for component governance scans. - cgIgnoreDirectories: [] - - # If set to true and running on a non-public project, - # Internal nuget and blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: true -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - cgIgnoreDirectories: ${{ parameters.cgIgnoreDirectories }} - enableInternalSources: ${{ parameters.enableInternalSources }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} - cgIgnoreDirectories: ${{ parameters.cgIgnoreDirectories }} - enableInternalSources: ${{ parameters.enableInternalSources }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index 173914f2364a..c32fc49233f8 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -1,22 +1,8 @@ variables: - - group: Publish-Build-Assets +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - - name: runCodesignValidationInjection - value: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 8b6f0cd8a43c..2364c0fd4a52 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -1,291 +1,8 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2022.amd64 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 - - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml index 3a56abf8922e..024397d87864 100644 --- a/eng/common/templates-official/post-build/setup-maestro-vars.yml +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -1,70 +1,8 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -UseBasicParsing -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml index cbba0596709d..30bb3985ca2b 100644 --- a/eng/common/templates-official/steps/component-governance.yml +++ b/eng/common/templates-official/steps/component-governance.yml @@ -1,13 +1,7 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml index 93a8394a666b..f9dd238c6cd5 100644 --- a/eng/common/templates-official/steps/enable-internal-runtimes.yml +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -1,28 +1,9 @@ # Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' # variable with the base64-encoded SAS token, by default - -parameters: -- name: federatedServiceConnection - type: string - default: 'dotnetbuilds-internal-read' -- name: outputVariableName - type: string - default: 'dotnetbuilds-internal-container-read-token-base64' -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: true - steps: -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates-official/steps/get-delegation-sas.yml - parameters: - federatedServiceConnection: ${{ parameters.federatedServiceConnection }} - outputVariableName: ${{ parameters.outputVariableName }} - expiryInHours: ${{ parameters.expiryInHours }} - base64Encode: ${{ parameters.base64Encode }} - storageAccount: dotnetbuilds - container: internal - permissions: rl +- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/enable-internal-sources.yml b/eng/common/templates-official/steps/enable-internal-sources.yml new file mode 100644 index 000000000000..e6d57182284d --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-sources.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml index 1536353566c7..9a89a4706d94 100644 --- a/eng/common/templates-official/steps/generate-sbom.yml +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -1,48 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 8.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: true + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml index c690cc0a070c..c5a9c1f8275c 100644 --- a/eng/common/templates-official/steps/get-delegation-sas.yml +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -1,52 +1,7 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: false -- name: storageAccount - type: string -- name: container - type: string -- name: permissions - type: string - default: 'rl' - steps: -- task: AzureCLI@2 - displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - # Calculate the expiration of the SAS token and convert to UTC - $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - - # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads - # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 - $sas = "" - do { - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - } while($sas.IndexOf('/') -ne -1) - - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - - if ('${{ parameters.base64Encode }}' -eq 'true') { - $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) - } +- template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + is1ESPipeline: true - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml index 55e33bd38f71..c8dcf6b81392 100644 --- a/eng/common/templates-official/steps/get-federated-access-token.yml +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -1,40 +1,7 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: stepName - type: string - default: 'getFederatedAccessToken' -- name: condition - type: string - default: '' -# Resource to get a token for. Common values include: -# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps -# - 'https://storage.azure.com/' for storage -# Defaults to Azure DevOps -- name: resource - type: string - default: '499b84ac-1321-427f-aa17-267ca6975798' -- name: isStepOutputVariable - type: boolean - default: false - steps: -- task: AzureCLI@2 - displayName: 'Getting federated access token for feeds' - name: ${{ parameters.stepName }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" - exit 1 - } - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file +- template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml new file mode 100644 index 000000000000..100a3fc98493 --- /dev/null +++ b/eng/common/templates-official/steps/publish-build-artifacts.yml @@ -0,0 +1,41 @@ +parameters: +- name: displayName + type: string + default: 'Publish to Build Artifact' + +- name: condition + type: string + default: succeeded() + +- name: artifactName + type: string + +- name: pathToPublish + type: string + +- name: continueOnError + type: boolean + default: false + +- name: publishLocation + type: string + default: 'Container' + +- name: is1ESPipeline + type: boolean + default: true + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + ${{ if parameters.continueOnError }}: + continueOnError: ${{ parameters.continueOnError }} + inputs: + PublishLocation: ${{ parameters.publishLocation }} + PathtoPublish: ${{ parameters.pathToPublish }} + ${{ if parameters.artifactName }}: + ArtifactName: ${{ parameters.artifactName }} + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml index af5a40b64c4b..579fd531e94c 100644 --- a/eng/common/templates-official/steps/publish-logs.yml +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -1,23 +1,7 @@ -parameters: - StageLabel: '' - JobLabel: '' - steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(System.DefaultWorkingDirectory)/artifacts/log/Debug/* $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: true -- task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml new file mode 100644 index 000000000000..172f9f0fdc97 --- /dev/null +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,28 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: true + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.properties }}: + properties: ${{ parameters.args.properties }} + ${{ if parameters.args.sbomEnabled }}: + sbomEnabled: ${{ parameters.args.sbomEnabled }} diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml index 83d97a26a01f..5594551508a3 100644 --- a/eng/common/templates-official/steps/retain-build.yml +++ b/eng/common/templates-official/steps/retain-build.yml @@ -1,28 +1,7 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml index 22f2501307d4..6500f21bf845 100644 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -1,92 +1,7 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - HelixProjectArguments: '' # optional -- arguments passed to the build command for helixpublish.proj - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index c307825c9122..8f92c49e7b06 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -1,135 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - - # Optional list of directories to ignore for component governance scans. - cgIgnoreDirectories: [] - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh - $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh $(System.DefaultWorkingDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(System.DefaultWorkingDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/source-build/self/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: true -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ${{ if eq(length(parameters.cgIgnoreDirectories), 0) }}: - ignoreDirectories: '$(System.DefaultWorkingDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' - ${{ else }}: - ignoreDirectories: ${{ join(',', parameters.cgIgnoreDirectories) }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 80454d5a5587..5bdd3dd85fd2 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,263 +1,82 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false +parameters: enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false disableComponentGovernance: '' componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false # Sbom related params enableSbom: true - PackageVersion: 7.0.0 + runAsPublic: false + PackageVersion: 9.0.0 BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} +- template: /eng/common/core-templates/job/job.yml + parameters: + is1ESPipeline: false - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} + ${{ each parameter in parameters }}: + ${{ if and(ne(parameter.key, 'steps'), ne(parameter.key, 'is1ESPipeline')) }}: + ${{ parameter.key }}: ${{ parameter.value }} - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} + steps: + - ${{ each step in parameters.steps }}: + - ${{ step }} - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea + componentGovernanceSteps: + - template: /eng/common/templates/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} - richNavLogOutputDirectory: $(System.DefaultWorkingDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false + disableComponentGovernance: true ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(System.DefaultWorkingDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + artifactPublishSteps: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + publishLocation: Container + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + sbomEnabled: false # we don't need SBOM for logs + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish Logs + pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + targetPath: '$(System.DefaultWorkingDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true + sbomEnabled: false # we don't need SBOM for BuildConfiguration diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 0d9a54751c81..ff829dc4c700 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -1,109 +1,7 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(System.DefaultWorkingDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: false - - task: PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9234ccab1a45..ab2edec2adb5 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,173 +1,7 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - - repositoryAlias: self - - officialBuildId: '' - jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/common-variables.yml - - name: OfficialBuildId - ${{ if ne(parameters.officialBuildId, '') }}: - value: ${{ parameters.officialBuildId }} - ${{ else }}: - value: $(Build.BuildNumber) - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: ${{ parameters.repositoryAlias }} - fetchDepth: 3 - clean: true - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Build Assets - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1 - arguments: > - -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(OfficialBuildId) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - - task: PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: false - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 97021335cfc4..e44d47b1d760 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -1,78 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - - # Optional list of directories to ignore for component governance scans. - cgIgnoreDirectories: [] - - # If set to true and running on a non-public project, - # Internal blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: false - steps: - - ${{ if eq(parameters.enableInternalSources, true) }}: - - template: /eng/common/templates/steps/enable-internal-runtimes.yml - - template: /eng/common/templates/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} - cgIgnoreDirectories: ${{ parameters.cgIgnoreDirectories }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 1975582b9b05..89f3291593cb 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,82 +1,7 @@ -parameters: - runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexUploadPackageVersion - value: ${{ parameters.sourceIndexUploadPackageVersion }} - - name: SourceIndexProcessBinlogPackageVersion - value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - displayName: Get stage 1 auth token - inputs: - azureSubscription: 'SourceDotNet Stage1 Publish' - addSpnToEnvironment: true - scriptType: 'ps' - scriptLocation: 'inlineScript' - inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - - - script: | - az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) - displayName: "Login to Azure" +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: false - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml index e8b43e3b4cba..517f24d6a52c 100644 --- a/eng/common/templates/jobs/codeql-build.yml +++ b/eng/common/templates/jobs/codeql-build.yml @@ -1,31 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - jobs: -- template: /eng/common/templates/jobs/jobs.yml +- template: /eng/common/core-templates/jobs/codeql-build.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true + is1ESPipeline: false - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 7eafc256758f..388e9037b3e6 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,101 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - repositoryAlias: self - officialBuildId: '' - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: false - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} - repositoryAlias: ${{ parameters.repositoryAlias }} - officialBuildId: ${{ parameters.officialBuildId }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index 4dde599add97..818d4c326dbb 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -1,59 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - - # Optional list of directories to ignore for component governance scans. - cgIgnoreDirectories: [] - - # If set to true and running on a non-public project, - # Internal nuget and blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: false -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - cgIgnoreDirectories: ${{ parameters.cgIgnoreDirectories }} - enableInternalSources: ${{ parameters.enableInternalSources }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} - cgIgnoreDirectories: ${{ parameters.cgIgnoreDirectories }} - enableInternalSources: ${{ parameters.enableInternalSources }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 173914f2364a..7fa105875592 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,22 +1,8 @@ variables: - - group: Publish-Build-Assets +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - - name: runCodesignValidationInjection - value: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index db4bf92b90b9..53ede714bdd2 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,287 +1,8 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} - publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }} - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 - - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 4347fa80b684..a79fab5b441e 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,70 +1,8 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: AzureCLI@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - . $(System.DefaultWorkingDirectory)\eng\common\tools.ps1 - $darc = Get-Darc - $buildInfo = & $darc get-build ` - --id ${{ parameters.BARBuildId }} ` - --extended ` - --output-format json ` - --ci ` - | convertFrom-Json - - $BarId = ${{ parameters.BARBuildId }} - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index cbba0596709d..c12a5f8d21d7 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -1,13 +1,7 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml index 54dc9416c519..b21a8038cc1c 100644 --- a/eng/common/templates/steps/enable-internal-runtimes.yml +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -1,28 +1,10 @@ # Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' # variable with the base64-encoded SAS token, by default -parameters: -- name: federatedServiceConnection - type: string - default: 'dotnetbuilds-internal-read' -- name: outputVariableName - type: string - default: 'dotnetbuilds-internal-container-read-token-base64' -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: true - steps: -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates/steps/get-delegation-sas.yml - parameters: - federatedServiceConnection: ${{ parameters.federatedServiceConnection }} - outputVariableName: ${{ parameters.outputVariableName }} - expiryInHours: ${{ parameters.expiryInHours }} - base64Encode: ${{ parameters.base64Encode }} - storageAccount: dotnetbuilds - container: internal - permissions: rl +- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/enable-internal-sources.yml b/eng/common/templates/steps/enable-internal-sources.yml new file mode 100644 index 000000000000..5f87e9abb8aa --- /dev/null +++ b/eng/common/templates/steps/enable-internal-sources.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index b1fe8b3944b3..26dc00a2e0f3 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -1,48 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 8.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml index c690cc0a070c..83760c9798e3 100644 --- a/eng/common/templates/steps/get-delegation-sas.yml +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -1,52 +1,7 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: false -- name: storageAccount - type: string -- name: container - type: string -- name: permissions - type: string - default: 'rl' - steps: -- task: AzureCLI@2 - displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - # Calculate the expiration of the SAS token and convert to UTC - $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - - # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads - # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 - $sas = "" - do { - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - } while($sas.IndexOf('/') -ne -1) - - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - - if ('${{ parameters.base64Encode }}' -eq 'true') { - $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) - } +- template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + is1ESPipeline: false - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml index 55e33bd38f71..31e151d9d9e7 100644 --- a/eng/common/templates/steps/get-federated-access-token.yml +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -1,40 +1,7 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: stepName - type: string - default: 'getFederatedAccessToken' -- name: condition - type: string - default: '' -# Resource to get a token for. Common values include: -# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps -# - 'https://storage.azure.com/' for storage -# Defaults to Azure DevOps -- name: resource - type: string - default: '499b84ac-1321-427f-aa17-267ca6975798' -- name: isStepOutputVariable - type: boolean - default: false - steps: -- task: AzureCLI@2 - displayName: 'Getting federated access token for feeds' - name: ${{ parameters.stepName }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" - exit 1 - } - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file +- template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml new file mode 100644 index 000000000000..6428a98dfef6 --- /dev/null +++ b/eng/common/templates/steps/publish-build-artifacts.yml @@ -0,0 +1,40 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: displayName + type: string + default: 'Publish to Build Artifact' + +- name: condition + type: string + default: succeeded() + +- name: artifactName + type: string + +- name: pathToPublish + type: string + +- name: continueOnError + type: boolean + default: false + +- name: publishLocation + type: string + default: 'Container' + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + ${{ if parameters.continueOnError }}: + continueOnError: ${{ parameters.continueOnError }} + inputs: + PublishLocation: ${{ parameters.publishLocation }} + PathtoPublish: ${{ parameters.pathToPublish }} + ${{ if parameters.artifactName }}: + ArtifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index e2f8413d8e19..4ea86bd88235 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,23 +1,7 @@ -parameters: - StageLabel: '' - JobLabel: '' - steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(System.DefaultWorkingDirectory)/artifacts/log/Debug/* $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: false -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(System.DefaultWorkingDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/publish-pipeline-artifacts.yml b/eng/common/templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 000000000000..5dd698b212fc --- /dev/null +++ b/eng/common/templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,34 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.publishLocation }}: + publishLocation: ${{ parameters.args.publishLocation }} + ${{ if parameters.args.fileSharePath }}: + fileSharePath: ${{ parameters.args.fileSharePath }} + ${{ if parameters.args.Parallel }}: + parallel: ${{ parameters.args.Parallel }} + ${{ if parameters.args.parallelCount }}: + parallelCount: ${{ parameters.args.parallelCount }} + ${{ if parameters.args.properties }}: + properties: ${{ parameters.args.properties }} \ No newline at end of file diff --git a/eng/common/templates/steps/retain-build.yml b/eng/common/templates/steps/retain-build.yml index 83d97a26a01f..8e841ace3d29 100644 --- a/eng/common/templates/steps/retain-build.yml +++ b/eng/common/templates/steps/retain-build.yml @@ -1,28 +1,7 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 22f2501307d4..39f99fc2762d 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,92 +1,7 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - HelixProjectArguments: '' # optional -- arguments passed to the build command for helixpublish.proj - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index d08a0e92caa4..23c1d6f4e9f8 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -1,135 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - - # Optional list of directories to ignore for component governance scans. - cgIgnoreDirectories: [] - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh - $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh $(System.DefaultWorkingDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(System.DefaultWorkingDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/source-build/self/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: false -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ${{ if eq(length(parameters.cgIgnoreDirectories), 0) }}: - ignoreDirectories: '$(System.DefaultWorkingDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' - ${{ else }}: - ignoreDirectories: ${{ join(',', parameters.cgIgnoreDirectories) }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 00a41b0d2a6f..18693ea120d5 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -24,34 +24,36 @@ # pool: # name: $(DncEngInternalBuildPool) # demands: ImageOverride -equals windows.vs2022.amd64 - variables: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ else }}: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - name: DncEngPublicBuildPool - value: $[ - replace( + # Any new -Svc alternative pools should have variables added here to allow for splitting work + - name: DncEngPublicBuildPool + value: $[ replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore-Svc-Public' - ), - False, - 'NetCore-Public' - ) - ] + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] - - name: DncEngInternalBuildPool - value: $[ - replace( + - name: DncEngInternalBuildPool + value: $[ replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ] + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 0acf27aa3aae..a06513a59407 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,6 +65,11 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } +# True if the build is a product build +[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } + +[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } + function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -158,18 +163,13 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_MULTILEVEL_LOOKUP=0 # Disable first run since we do not need all ASP.NET packages restored. - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + $env:DOTNET_NOLOGO=1 # Disable telemetry on CI. if ($ci) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 } - # Source Build uses DotNetCoreSdkDir variable - if ($env:DotNetCoreSdkDir -ne $null) { - $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir - } - # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetExecutable = GetExecutableFileName 'dotnet' @@ -228,7 +228,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' - Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -254,7 +254,6 @@ function Retry($downloadBlock, $maxRetries = 5) { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." break } - } } @@ -384,7 +383,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.12.0 + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.12.0 $defaultXCopyMSBuildVersion = '17.12.0' if (!$vsRequirements) { @@ -424,7 +423,6 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] @@ -450,7 +448,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." + throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." } } if ($vsInstallDir -eq $null) { @@ -487,7 +485,7 @@ function InstallXCopyMSBuild([string]$packageVersion) { } function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'RoslynTools.MSBuild' + $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" @@ -504,6 +502,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath }) + if (!(Test-Path $packagePath)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" + throw + } Unzip $packagePath $packageDir } @@ -611,11 +613,11 @@ function InitializeBuildTool() { # Use override if it exists - commonly set by source-build if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { - $initializeBuildToolFramework="net8.0" + $initializeBuildToolFramework="net9.0" } else { $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework } - + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } } elseif ($msbuildEngine -eq "vs") { try { @@ -658,7 +660,7 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOCACHE = $true + $env:RESTORENOHTTPCACHE = $true } } @@ -691,8 +693,14 @@ function Read-ArcadeSdkVersion() { } function InitializeToolset() { - if (Test-Path variable:global:_ToolsetBuildProj) { - return $global:_ToolsetBuildProj + # For Unified Build/Source-build support, check whether the environment variable is + # set. If it is, then use this as the toolset build project. + if ($env:_InitializeToolset -ne $null) { + return $global:_InitializeToolset = $env:_InitializeToolset + } + + if (Test-Path variable:global:_InitializeToolset) { + return $global:_InitializeToolset } $nugetCache = GetNuGetPackageCachePath @@ -703,7 +711,7 @@ function InitializeToolset() { if (Test-Path $toolsetLocationFile) { $path = Get-Content $toolsetLocationFile -TotalCount 1 if (Test-Path $path) { - return $global:_ToolsetBuildProj = $path + return $global:_InitializeToolset = $path } } @@ -726,7 +734,7 @@ function InitializeToolset() { throw "Invalid toolset path: $path" } - return $global:_ToolsetBuildProj = $path + return $global:_InitializeToolset = $path } function ExitWithExitCode([int] $exitCode) { @@ -778,12 +786,10 @@ function MSBuild() { # new scripts need to work with old packages, so we need to look for the old names/versions (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) - (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll')) (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')) + (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')), + (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll')) ) $selectedPath = $null foreach ($path in $possiblePaths) { @@ -842,7 +848,8 @@ function MSBuild-Core() { } } - $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + # Be sure quote the path in case there are spaces in the dotnet installation location. + $env:ARCADE_BUILD_TOOL_COMMAND = "`"$($buildTool.Path)`" $cmdArgs" $exitCode = Exec-Process $buildTool.Path $cmdArgs @@ -857,7 +864,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) { + # Skip this when the build is a child of the VMR orchestrator build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 68db15430230..01b09b65796c 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,6 +68,9 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} +# True if the build is a product build +product_build=${product_build:-false} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -112,7 +115,7 @@ function InitializeDotNetCli { export DOTNET_MULTILEVEL_LOOKUP=0 # Disable first run since we want to control all package sources - export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + export DOTNET_NOLOGO=1 # Disable telemetry on CI if [[ $ci == true ]]; then @@ -123,11 +126,6 @@ function InitializeDotNetCli { # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" - # Source Build uses DotNetCoreSdkDir variable - if [[ -n "${DotNetCoreSdkDir:-}" ]]; then - export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" - fi - # Find the first path on $PATH that contains the dotnet.exe if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` @@ -146,7 +144,7 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="$repo_root/.dotnet" + dotnet_root="${repo_root}.dotnet" export DOTNET_INSTALL_DIR="$dotnet_root" @@ -165,7 +163,7 @@ function InitializeDotNetCli { Write-PipelinePrependPath -path "$dotnet_root" Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" - Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" + Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" # return value _InitializeDotNetCli="$dotnet_root" @@ -310,7 +308,7 @@ function GetDotNetInstallScript { curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { if command -v openssl &> /dev/null; then echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" - echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 || true fi echo "Will now retry the same URL with verbose logging." with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { @@ -343,20 +341,20 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" # use override if it exists - commonly set by source-build if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then - _InitializeBuildToolFramework="net8.0" + _InitializeBuildToolFramework="net9.0" else _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" fi } -# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 +# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then - export NUGET_PACKAGES="$HOME/.nuget/packages" + export NUGET_PACKAGES="$HOME/.nuget/packages/" else - export NUGET_PACKAGES="$repo_root/.packages" - export RESTORENOCACHE=true + export NUGET_PACKAGES="$repo_root/.packages/" + export RESTORENOHTTPCACHE=true fi fi @@ -440,7 +438,7 @@ function StopProcesses { } function MSBuild { - local args=$@ + local args=( "$@" ) if [[ "$pipelines_log" == true ]]; then InitializeBuildTool InitializeToolset @@ -458,12 +456,10 @@ function MSBuild { local possiblePaths=() possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" ) possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" ) + possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" ) for path in "${possiblePaths[@]}"; do if [[ -f $path ]]; then selectedPath=$path @@ -477,7 +473,7 @@ function MSBuild { args+=( "-logger:$selectedPath" ) fi - MSBuild-Core ${args[@]} + MSBuild-Core "${args[@]}" } function MSBuild-Core { @@ -510,7 +506,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then + # Skip this when the build is a child of the VMR orchestrator build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/global.json b/global.json index 8e2e19db94a0..50548f9d41c0 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "tools": { - "dotnet": "8.0.126", + "dotnet": "9.0.115", "runtimes": { "dotnet": [ - "$(VSRedistCommonNetCoreSharedFrameworkx6480PackageVersion)" + "$(VSRedistCommonNetCoreSharedFrameworkx6490PackageVersion)" ], "aspnetcore": [ "$(MicrosoftAspNetCoreComponentsSdkAnalyzersPackageVersion)" @@ -13,8 +13,13 @@ "version": "16.8" } }, + "native-tools": { + "cmake": "latest" + }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26224.3", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.26224.3" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26220.3", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26220.3", + "Microsoft.Build.NoTargets": "3.7.0", + "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } } From 11c70475d8d108768b5100642e9aa078e273b1a3 Mon Sep 17 00:00:00 2001 From: ProductConstructionServiceProd Date: Tue, 19 May 2026 23:55:12 +0000 Subject: [PATCH 21/29] Merged PR 61302: [internal/release/9.0.3xx] Update dependencies from dnceng/internal/dotnet-aspnetcore This pull request updates the following dependencies [marker]: <> (Begin:fccb84dc-5ab4-47fa-89b4-a043eabf59b7) ## From https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - **Subscription**: [fccb84dc-5ab4-47fa-89b4-a043eabf59b7](https://maestro.dot.net/subscriptions?search=fccb84dc-5ab4-47fa-89b4-a043eabf59b7) - **Build**: [20260519.7](https://dev.azure.com/dnceng/internal/_build/results?buildId=2979108) ([315055](https://maestro.dot.net/channel/5128/azdo:dnceng:internal:dotnet-aspnetcore/build/315055)) - **Date Produced**: May 19, 2026 9:31:40 PM UTC - **Commit**: [5edf41b9c09952f18e404ad38e25670991c1b513](https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore?_a=history&version=GC5edf41b9c09952f18e404ad38e25670991c1b513) - **Branch**: [refs/heads/internal/release/9.0](https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore?version=GBrefs/heads/internal/release/9.0) [DependencyUpdate]: <> (Begin) - **Dependency Updates**: - From [9.0.17-servicing.26268.2 to 9.0.17-servicing.26269.7][1] - dotnet-dev-certs - dotnet-user-jwts - dotnet-user-secrets - Microsoft.AspNetCore.Analyzers - Microsoft.AspNetCore.App.Ref.Internal - Microsoft.AspNetCore.Components.SdkAnalyzers - Microsoft.AspNetCore.DeveloperCertificates.XPlat - Microsoft.AspNetCore.Mvc.Analyzers - Microsoft.AspNetCore.Mvc.Api.Analyzers - VS.Redist.Common.AspNetCore.SharedFramework.x64.9.0 - Microsoft.SourceBuild.Intermediate.aspnetcore - From [9.0.17 to 9.0.17][1] - Microsoft.AspNetCore.App.Ref - Microsoft.AspNetCore.App.Runtime.win-x64 - Microsoft.AspNetCore.Authorization - Microsoft.AspNetCore.Components.Web - Microsoft.AspNetCore.TestHost - Microsoft.Extensions.FileProviders.Embedded - Microsoft.Extensions.ObjectPool - Microsoft.JSInterop [1]: https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore/branches?baseVersion=GC5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2&targetVersion=GC5edf41b9c09952f18e404ad38e25670991c1b513&_a=files [DependencyUpdate]: <> (End) [marker]: <> (End:fccb84dc-5ab4-47fa-89b4-a043eabf59b7) --- NuGet.config | 4 +-- eng/Version.Details.xml | 60 ++++++++++++++++++++--------------------- eng/Versions.props | 20 +++++++------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/NuGet.config b/NuGet.config index 24dacdb6230c..f9a1a606c2bd 100644 --- a/NuGet.config +++ b/NuGet.config @@ -27,7 +27,7 @@ - + @@ -72,7 +72,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5aebec0e1f6f..1f8f5181ef4a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -131,13 +131,13 @@ https://github.com/dotnet/roslyn 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted @@ -273,52 +273,52 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 @@ -341,19 +341,19 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 @@ -507,7 +507,7 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5d6e3342e5d3b1d82f5ac41a027c00ae56f31ea2 + 5edf41b9c09952f18e404ad38e25670991c1b513 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index 241868f446e1..c4837cc826d8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -218,18 +218,18 @@ 9.0.17 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 9.0.17 9.0.17 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 - 9.0.17-servicing.26268.2 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 + 9.0.17-servicing.26269.7 From d2866891db7a6cc573b04714712d4786b7a28eed Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 28 May 2026 02:01:26 +0000 Subject: [PATCH 22/29] Update dependencies from https://github.com/dotnet/scenario-tests build 20260527.1 On relative base path root Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.26263.1 -> To Version 9.0.0-preview.26277.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3ec2ea1c8b31..e996bff07146 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -597,14 +597,14 @@ https://github.com/dotnet/arcade-services e156e649f28395d9d0ee1e848225a689b59e0fd3 - + https://github.com/dotnet/scenario-tests - d130a15593a5a26d1c35da3d4204e35bf99e1e9d + f5c3e985fbb774814d37d224e2a9e06888cdc06d - + https://github.com/dotnet/scenario-tests - d130a15593a5a26d1c35da3d4204e35bf99e1e9d + f5c3e985fbb774814d37d224e2a9e06888cdc06d From 8fb85bc46a626338603c3aee09423aa48d881830 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 2 Jun 2026 02:02:06 +0000 Subject: [PATCH 23/29] Update dependencies from https://github.com/dotnet/scenario-tests build 20260601.3 On relative base path root Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.26263.1 -> To Version 9.0.0-preview.26301.3 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e996bff07146..3fd5a9a63e70 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -597,14 +597,14 @@ https://github.com/dotnet/arcade-services e156e649f28395d9d0ee1e848225a689b59e0fd3 - + https://github.com/dotnet/scenario-tests - f5c3e985fbb774814d37d224e2a9e06888cdc06d + d1e1e7918f07c3ca9e5eb2f617159f27a4d2ec28 - + https://github.com/dotnet/scenario-tests - f5c3e985fbb774814d37d224e2a9e06888cdc06d + d1e1e7918f07c3ca9e5eb2f617159f27a4d2ec28 From 5ba6534b7b46ccccd47d30b689f69f8675d2d4bb Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 2 Jun 2026 11:32:37 -0700 Subject: [PATCH 24/29] Update branding to 9.0.119 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 457c83a51328..4e2432c1c324 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,7 +6,7 @@ 9 0 1 - 18 + 19 From d9ddf8c5753a664b8d42a2ef39d55fcadba987e0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 3 Jun 2026 02:02:07 +0000 Subject: [PATCH 25/29] Update dependencies from https://github.com/dotnet/scenario-tests build 20260602.1 On relative base path root Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.26263.1 -> To Version 9.0.0-preview.26302.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3fd5a9a63e70..0c7fd57b18f1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -597,14 +597,14 @@ https://github.com/dotnet/arcade-services e156e649f28395d9d0ee1e848225a689b59e0fd3 - + https://github.com/dotnet/scenario-tests - d1e1e7918f07c3ca9e5eb2f617159f27a4d2ec28 + 623d5a21e18d6d1f370b5b9d1ebaf8d687464c0b - + https://github.com/dotnet/scenario-tests - d1e1e7918f07c3ca9e5eb2f617159f27a4d2ec28 + 623d5a21e18d6d1f370b5b9d1ebaf8d687464c0b From 34346a98c8ca094a2ad5cb3aa3844a7512ff4cd6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:03:37 +0000 Subject: [PATCH 26/29] Reset files to release/9.0.3xx Reset patterns: - global.json - NuGet.config - eng/Version.Details.xml - eng/Versions.props - eng/common/* --- NuGet.config | 10 +- eng/Version.Details.xml | 256 +++++++++++++++++----------------------- eng/Versions.props | 103 ++++++++-------- global.json | 6 +- 4 files changed, 172 insertions(+), 203 deletions(-) diff --git a/NuGet.config b/NuGet.config index dcf574945f80..7eee81cab369 100644 --- a/NuGet.config +++ b/NuGet.config @@ -27,15 +27,17 @@ + + - + - + @@ -61,11 +63,11 @@ + + - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 914fc315ca9c..3ec2ea1c8b31 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,18 +1,18 @@ - + https://github.com/dotnet/templating - c96cefdd04b806246127b9ad69dcb60c31b56e6f + 1ad30c339011d797841a203de0a27c5106bad25f - + https://github.com/dotnet/templating - c96cefdd04b806246127b9ad69dcb60c31b56e6f + 1ad30c339011d797841a203de0a27c5106bad25f - + https://github.com/dotnet/templating - c96cefdd04b806246127b9ad69dcb60c31b56e6f + b73682307aa0128c5edbec94c2e6a070d13ae6bb @@ -59,10 +59,6 @@ https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - - https://github.com/dotnet/emsdk - e2909c00ead6fb5d18a5167ca78f259c639084e0 - https://github.com/dotnet/emsdk e2909c00ead6fb5d18a5167ca78f259c639084e0 @@ -73,67 +69,67 @@ e2909c00ead6fb5d18a5167ca78f259c639084e0 - + https://github.com/dotnet/msbuild - 07da1b9a89da6d00c5a5a6a385cdcebfdfed7110 + 2a0eb78b31025538b6e749ecd80dfabfcfa43a42 - + https://github.com/dotnet/msbuild - 07da1b9a89da6d00c5a5a6a385cdcebfdfed7110 + 2a0eb78b31025538b6e749ecd80dfabfcfa43a42 - + https://github.com/dotnet/msbuild - 07da1b9a89da6d00c5a5a6a385cdcebfdfed7110 + 2a0eb78b31025538b6e749ecd80dfabfcfa43a42 - + https://github.com/dotnet/fsharp - 47d4e3f91e4e5414b6dafbf14288b9c5a798ef99 + 14987c804f33917bf15f4c25e0cd16ecd01807f4 - + https://github.com/dotnet/fsharp - 47d4e3f91e4e5414b6dafbf14288b9c5a798ef99 + 14987c804f33917bf15f4c25e0cd16ecd01807f4 - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn - fc52718eccdb37693a40a518b1178b1e23114e68 + 450493a9b4ec6337bced0120e97cb76f4ed783db https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -143,91 +139,91 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore 4d825aeb5e5023588c036709c7914008b625b0eb - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - c92125011405028c945371a89e1a1eb0e735456d + 13eeb32d62833984c9367f8f4e22a120ebc03b47 - + https://github.com/microsoft/vstest - bc9161306b23641b0364b8f93d546da4d48da1eb + 51441adcd6c424ae7315d66ce7e96baf34d70369 - + https://github.com/microsoft/vstest - bc9161306b23641b0364b8f93d546da4d48da1eb + 51441adcd6c424ae7315d66ce7e96baf34d70369 - + https://github.com/microsoft/vstest - bc9161306b23641b0364b8f93d546da4d48da1eb + 51441adcd6c424ae7315d66ce7e96baf34d70369 - + https://github.com/microsoft/vstest - bc9161306b23641b0364b8f93d546da4d48da1eb + 51441adcd6c424ae7315d66ce7e96baf34d70369 @@ -325,22 +321,22 @@ 4d825aeb5e5023588c036709c7914008b625b0eb - + https://github.com/dotnet/razor - cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 + 41f3afd466695ac2460260431537fe4d779ff446 - + https://github.com/dotnet/razor - cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 + 41f3afd466695ac2460260431537fe4d779ff446 - + https://github.com/dotnet/razor - cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 + 41f3afd466695ac2460260431537fe4d779ff446 - + https://github.com/dotnet/razor - cff92f3cc3f19a607ddbb7a0cddfbccf87a1c061 + 41f3afd466695ac2460260431537fe4d779ff446 @@ -359,36 +355,6 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore 4d825aeb5e5023588c036709c7914008b625b0eb - - https://github.com/dotnet/test-templates - 0385265f4d0b6413d64aea0223172366a9b9858c - - - https://github.com/dotnet/test-templates - 307b8f538d83a955d8f6dd909eee41a5555f2f4d - - - https://github.com/dotnet/test-templates - becc4bd157cd6608b51a5ffe414a5d2de6330272 - - - https://github.com/dotnet/test-templates - becc4bd157cd6608b51a5ffe414a5d2de6330272 - - - https://github.com/dotnet/test-templates - 49c9ad01f057b3c6352bbec12b117acc2224493c - - - https://github.com/dotnet/test-templates - 47c90e140b027225b799ca8413af10ee3d5f1126 - - - - https://github.com/dotnet/test-templates - 47c90e140b027225b799ca8413af10ee3d5f1126 - - https://dev.azure.com/dnceng/internal/_git/dotnet-winforms @@ -408,18 +374,18 @@ 63ae81154c50a1cf9287cc47d8351d55b4289e6d - + https://github.com/dotnet/roslyn-analyzers - b9b54526b7908ea519b503196100a34dd2e52374 + 94e92044351f9eb9e75b22f6000647393e36f5b9 - - https://github.com/dotnet/roslyn-analyzers - b9b54526b7908ea519b503196100a34dd2e52374 + + https://github.com/dotnet/roslyn + 450493a9b4ec6337bced0120e97cb76f4ed783db - + https://github.com/dotnet/roslyn-analyzers - b9b54526b7908ea519b503196100a34dd2e52374 + 94e92044351f9eb9e75b22f6000647393e36f5b9 @@ -445,16 +411,16 @@ - + https://github.com/dotnet/source-build-externals - 16c380d1ce5fa0b24e232251c31cb013bbf3365f + 71dbdccd13f28cfd1a35649263b55ebbeab26ee7 - - https://github.com/dotnet/source-build-assets - 3a687f7154d73a6425d98a1deeb40424bbd7cb38 - + + https://github.com/dotnet/source-build-reference-packages + 6092b62b7f35fddbd6bf31e19b2ab64bbe2443ae + https://github.com/dotnet/deployment-tools @@ -589,34 +555,34 @@ - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 - + https://github.com/dotnet/arcade - f843e65cdfc9e9af05d987f2f7f2e6f6a6106eb5 + 6c1a2a69259c3f66af6176c9c70021b3d9989504 diff --git a/eng/Versions.props b/eng/Versions.props index 4e2432c1c324..c6880ec685f5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -5,8 +5,8 @@ 9 0 - 1 - 19 + 3 + 15 @@ -18,15 +18,15 @@ true release - rtm + preview rtm servicing - - + 0 true + true 6.0.1 @@ -35,7 +35,7 @@ 17 36 20 - $([MSBuild]::Add($(VersionFeature), 10)) + $([MSBuild]::Add($(VersionFeature), 13)) <_NET70ILLinkPackVersion>7.0.100-1.23211.1 @@ -76,13 +76,12 @@ 1.1.0-beta.25317.4 - - 9.0.16-servicing.26230.2 + + 9.1.0-preview.1.24555.3 - - - 1.1.0-rtm.25262.1 + + 9.0.16-servicing.26230.2 @@ -124,7 +123,10 @@ 9.0.16 + 4.5.1 + 4.5.5 8.0.5 + 4.5.4 9.0.16 9.0.16 @@ -137,29 +139,29 @@ - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 - 6.12.5-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 + 6.14.3-rc.1 - 17.12.0-release-24508-01 - 17.12.0-release-24508-01 - 17.12.0-release-24508-01 + 17.14.1-release-25428-01 + 17.14.1-release-25428-01 + 17.14.1-release-25428-01 - 9.0.0-preview.26226.5 - 3.11.0-beta1.26226.5 + 9.0.0-preview.26255.2 + 3.12.0-beta1.26064.1 @@ -170,8 +172,8 @@ Some .NET Framework tasks and the resolver will need to run in a VS/MSBuild that is older than the very latest, based on what we want the SDK to support. So use a version that matches the version - in minimumMSBuildVersion. Note that MSBuild has started versioning before release so the version we use as the Minimum should be .0 - to ensure we load in VS but the version we build against should be the version of MSBuild that ships in the .0 VS release. + in minimumMSBuildVersion. Note that MSBuild has started versioning before release so the version we use as the Minimum should be .0 + to ensure we load in VS but the version we build against should be the version of MSBuild that ships in the .0 VS release. In these cases, we don't want to use MicrosoftBuildVersion and other associated properties that are updated by the VMR infrastructure. So, we read this version from the 'minimumMSBuildVersion' file in non-source-only cases into MicrosoftBuildMinimumVersion, @@ -180,38 +182,38 @@ At usage sites, either we use MicrosoftBuildMinimumVersion, or MicrosoftBuildVersion in source-only modes. Additionally, set the MinimumVSVersion for the installer UI that's required for targeting NetCurrent --> - 17.12.57 - 17.12.57-preview-26069-01 + 17.14.43 + 17.14.43-servicing-26070-01 17.11.48 17.12 - 9.0.118 + 9.0.315 $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.118-servicing.26267.4 + 9.0.315-servicing.26268.1 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) - 12.9.101-beta.25070.7 + 13.9.303-beta.25361.1 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 - 4.12.0-3.25609.5 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 + 4.14.0-3.26064.1 @@ -231,9 +233,9 @@ - 9.0.0-preview.26069.1 - 9.0.0-preview.26069.1 - 9.0.0-preview.26069.1 + 9.0.0-preview.26064.2 + 9.0.0-preview.26064.2 + 9.0.0-preview.26064.2 @@ -270,10 +272,10 @@ - 9.0.0-beta.26220.3 - 9.0.0-beta.26220.3 - 9.0.0-beta.26220.3 - 9.0.0-beta.26220.3 + 9.0.0-beta.26261.1 + 9.0.0-beta.26261.1 + 9.0.0-beta.26261.1 + 9.0.0-beta.26261.1 @@ -317,7 +319,6 @@ 15.0.9617 18.0.9617 - 9.0.16-servicing.26221.3 9.0.16 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100PackageVersion) diff --git a/global.json b/global.json index 50548f9d41c0..5afb31ceeff1 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "9.0.115", + "dotnet": "9.0.116", "runtimes": { "dotnet": [ "$(VSRedistCommonNetCoreSharedFrameworkx6490PackageVersion)" @@ -17,8 +17,8 @@ "cmake": "latest" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26220.3", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26220.3", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26261.1", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26261.1", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } From 2ceba4815659ccdb1bcaafeebf8e17e4fdc85e02 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 06:07:36 +0000 Subject: [PATCH 27/29] [release/9.0.3xx] Update dependencies from dotnet/arcade (#54452) [release/9.0.3xx] Update dependencies from dotnet/arcade - Merge branch 'release/9.0.3xx' into darc-release/9.0.3xx-fd97d203-de32-409e-9c3d-b489d5871023 - Merge branch 'release/9.0.3xx' into darc-release/9.0.3xx-fd97d203-de32-409e-9c3d-b489d5871023 - Skip diag verbosity test variant that hangs testhost on FullFramework The ItUsesVerbosityPassedToDefineVerbosityOfConsoleLoggerOfTheTests test with verbosity 'diag' produces ~230K lines of MSBuild diagnostic output. When the inner build fails, FluentAssertions attempts to include the entire StdOut in the failure message, causing the testhost to hang and triggering a 15-minute blame-hang timeout. Fixes: https://github.com/dotnet/sdk/issues/54781 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/Version.Details.xml | 28 +++++++++---------- eng/Versions.props | 8 +++--- global.json | 4 +-- ...enDotnetTestBuildsAndRunsTestfromCsproj.cs | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0077be6e7ba8..e40900e9ac88 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -555,34 +555,34 @@ - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - + https://github.com/dotnet/arcade - 6c1a2a69259c3f66af6176c9c70021b3d9989504 + efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 diff --git a/eng/Versions.props b/eng/Versions.props index 5c6a6ffa706c..22268a0732ca 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -272,10 +272,10 @@ - 9.0.0-beta.26261.1 - 9.0.0-beta.26261.1 - 9.0.0-beta.26261.1 - 9.0.0-beta.26261.1 + 9.0.0-beta.26301.4 + 9.0.0-beta.26301.4 + 9.0.0-beta.26301.4 + 9.0.0-beta.26301.4 diff --git a/global.json b/global.json index 5afb31ceeff1..1825b53e035e 100644 --- a/global.json +++ b/global.json @@ -17,8 +17,8 @@ "cmake": "latest" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26261.1", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26261.1", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26301.4", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26301.4", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs index cc48a9932b0a..4fe87eda70c7 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs @@ -338,7 +338,7 @@ public void ItBuildsAndTestsAppWhenRestoringToSpecificDirectory() [InlineData("m", false)] [InlineData("n", true)] [InlineData("d", true)] - [InlineData("diag", true)] + [InlineData("diag", true, Skip = "https://github.com/dotnet/sdk/issues/54781")] public void ItUsesVerbosityPassedToDefineVerbosityOfConsoleLoggerOfTheTests(string verbosity, bool shouldShowPassedTests) { // Copy and restore VSTestCore project in output directory of project dotnet-vstest.Tests From eaa1d238114d5c9a1809cee816c0263f82ddabf7 Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Mon, 6 Jul 2026 13:23:24 -0700 Subject: [PATCH 28/29] Update branding on release/9.0.3xx --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 22268a0732ca..0ee734f78674 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,7 +6,7 @@ 9 0 3 - 16 + 17 From 31370094524c5b1ce0b80bd208f34d1300591478 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:36:34 +0000 Subject: [PATCH 29/29] Reset files to release/10.0.1xx Reset patterns: - global.json - NuGet.config - eng/Version.Details.xml - eng/Version.Details.props - eng/common/* - src/SourceBuild/* --- NuGet.config | 59 +- eng/Version.Details.props | 292 +++++ eng/Version.Details.xml | 1085 ++++++++--------- eng/common/CIBuild.cmd | 2 +- eng/common/SetupNugetSources.ps1 | 90 +- eng/common/SetupNugetSources.sh | 192 +-- eng/common/build.ps1 | 11 +- eng/common/build.sh | 33 +- eng/common/cibuild.sh | 2 +- eng/common/core-templates/job/job.yml | 53 +- eng/common/core-templates/job/onelocbuild.yml | 39 +- .../job/publish-build-assets.yml | 93 +- .../core-templates/job/source-build.yml | 15 +- .../job/source-index-stage1.yml | 47 +- .../core-templates/jobs/codeql-build.yml | 1 - eng/common/core-templates/jobs/jobs.yml | 20 +- .../core-templates/jobs/source-build.yml | 23 +- .../core-templates/post-build/post-build.yml | 147 ++- .../post-build/setup-maestro-vars.yml | 5 +- .../steps/cleanup-microbuild.yml | 28 + .../core-templates/steps/generate-sbom.yml | 60 +- .../steps/get-delegation-sas.yml | 11 +- .../steps/install-microbuild.yml | 110 ++ .../core-templates/steps/publish-logs.yml | 18 +- .../core-templates/steps/source-build.yml | 90 +- .../steps/source-index-stage1-publish.yml | 35 + eng/common/cross/arm64/tizen/tizen.patch | 2 +- eng/common/cross/build-android-rootfs.sh | 49 +- eng/common/cross/build-rootfs.sh | 364 ++++-- eng/common/cross/install-debs.py | 409 +++++++ eng/common/cross/tizen-build-rootfs.sh | 0 eng/common/cross/tizen-fetch.sh | 9 +- eng/common/cross/toolchain.cmake | 118 +- eng/common/darc-init.ps1 | 8 +- eng/common/darc-init.sh | 6 +- eng/common/dotnet.cmd | 7 + eng/common/dotnet.ps1 | 12 + eng/common/dotnet.sh | 26 + eng/common/generate-locproject.ps1 | 49 +- .../native/LocateNativeCompiler.targets | 27 + eng/common/native/NativeAotSupported.props | 26 + eng/common/native/init-distro-rid.sh | 2 + eng/common/native/install-dependencies.sh | 65 + eng/common/post-build/publish-using-darc.ps1 | 9 +- eng/common/post-build/redact-logs.ps1 | 9 +- eng/common/sdk-task.ps1 | 14 +- eng/common/sdk-task.sh | 121 ++ eng/common/sdl/packages.config | 2 +- eng/common/template-guidance.md | 2 - eng/common/templates-official/job/job.yml | 62 +- .../steps/publish-build-artifacts.yml | 7 +- .../steps/publish-pipeline-artifacts.yml | 4 +- .../steps/source-index-stage1-publish.yml | 7 + .../variables/pool-providers.yml | 2 +- eng/common/templates/job/job.yml | 57 +- .../steps/publish-build-artifacts.yml | 8 +- .../steps/source-index-stage1-publish.yml | 7 + eng/common/templates/steps/vmr-sync.yml | 186 +++ .../templates/variables/pool-providers.yml | 2 +- eng/common/templates/vmr-build-pr.yml | 43 + eng/common/tools.ps1 | 71 +- eng/common/tools.sh | 81 +- eng/common/vmr-sync.ps1 | 164 +++ eng/common/vmr-sync.sh | 227 ++++ global.json | 21 +- 65 files changed, 3365 insertions(+), 1481 deletions(-) create mode 100644 eng/Version.Details.props mode change 100644 => 100755 eng/common/SetupNugetSources.sh create mode 100644 eng/common/core-templates/steps/cleanup-microbuild.yml create mode 100644 eng/common/core-templates/steps/install-microbuild.yml create mode 100644 eng/common/core-templates/steps/source-index-stage1-publish.yml create mode 100755 eng/common/cross/install-debs.py mode change 100644 => 100755 eng/common/cross/tizen-build-rootfs.sh mode change 100644 => 100755 eng/common/cross/tizen-fetch.sh create mode 100644 eng/common/dotnet.cmd create mode 100644 eng/common/dotnet.ps1 create mode 100755 eng/common/dotnet.sh create mode 100644 eng/common/native/LocateNativeCompiler.targets create mode 100644 eng/common/native/NativeAotSupported.props create mode 100755 eng/common/native/install-dependencies.sh create mode 100755 eng/common/sdk-task.sh create mode 100644 eng/common/templates-official/steps/source-index-stage1-publish.yml create mode 100644 eng/common/templates/steps/source-index-stage1-publish.yml create mode 100644 eng/common/templates/steps/vmr-sync.yml create mode 100644 eng/common/templates/vmr-build-pr.yml create mode 100755 eng/common/vmr-sync.ps1 create mode 100755 eng/common/vmr-sync.sh diff --git a/NuGet.config b/NuGet.config index c096634414bd..3e32d4c1a1ee 100644 --- a/NuGet.config +++ b/NuGet.config @@ -3,48 +3,15 @@ - - - - - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - @@ -55,6 +22,8 @@ + + @@ -64,22 +33,14 @@ + + + - - - - - - - - - - - diff --git a/eng/Version.Details.props b/eng/Version.Details.props new file mode 100644 index 000000000000..e6a74a95f7f3 --- /dev/null +++ b/eng/Version.Details.props @@ -0,0 +1,292 @@ + + + + + 2.1.0 + + 10.0.10-servicing.26351.110 + 10.0.10-servicing.26351.110 + 10.0.10-servicing.26351.110 + 10.0.10-servicing.26351.110 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.10-servicing.26351.110 + 10.0.0-preview.26351.110 + 10.0.10 + 10.0.10 + 18.0.11 + 18.0.11-servicing-26351-110 + 7.0.3-rc.35210 + 10.0.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 10.0.0-preview.26351.110 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 2.0.0-rtm.1.26351.110 + 2.2.10 + 10.0.0-beta.26351.110 + 10.0.0-beta.26351.110 + 10.0.0-beta.26351.110 + 10.0.0-beta.26351.110 + 10.0.0-beta.26351.110 + 10.0.0-beta.26351.110 + 10.0.10 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.10-servicing.26351.110 + 10.0.0-beta.26351.110 + 10.0.0-beta.26351.110 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 14.0.110-servicing.26351.110 + 10.0.10 + 10.0.10 + 5.0.0-2.26351.110 + 5.0.0-2.26351.110 + 10.0.10-servicing.26351.110 + 10.0.10 + 10.0.10 + 10.0.0-preview.7.25377.103 + 10.0.0-preview.26351.110 + 10.0.10-servicing.26351.110 + 18.0.2-release-26351-110 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.110 + 10.0.110 + 10.0.110 + 10.0.110 + 10.0.110 + 10.0.110 + 10.0.110 + 10.0.110 + 10.0.110-servicing.26351.110 + 10.0.110 + 10.0.110-servicing.26351.110 + 10.0.110 + 10.0.110 + 10.0.110-servicing.26351.110 + 18.0.2-release-26351-110 + 18.0.2-release-26351-110 + 3.2.10 + 10.0.10 + 10.0.10-servicing.26351.110 + 10.0.10 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 7.0.3-rc.35210 + 10.0.10 + 2.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + 10.0.10 + + 2.3.0-preview.26203.3 + 4.3.0-preview.26203.3 + + + + + $(NETStandardLibraryRefPackageVersion) + + $(dotnetdevcertsPackageVersion) + $(dotnetuserjwtsPackageVersion) + $(dotnetusersecretsPackageVersion) + $(MicrosoftAspNetCoreAnalyzersPackageVersion) + $(MicrosoftAspNetCoreAppRefPackageVersion) + $(MicrosoftAspNetCoreAppRefInternalPackageVersion) + $(MicrosoftAspNetCoreAuthenticationFacebookPackageVersion) + $(MicrosoftAspNetCoreAuthenticationGooglePackageVersion) + $(MicrosoftAspNetCoreAuthenticationMicrosoftAccountPackageVersion) + $(MicrosoftAspNetCoreAuthorizationPackageVersion) + $(MicrosoftAspNetCoreComponentsPackageVersion) + $(MicrosoftAspNetCoreComponentsAnalyzersPackageVersion) + $(MicrosoftAspNetCoreComponentsFormsPackageVersion) + $(MicrosoftAspNetCoreComponentsSdkAnalyzersPackageVersion) + $(MicrosoftAspNetCoreComponentsWebPackageVersion) + $(MicrosoftAspNetCoreComponentsWebAssemblyPackageVersion) + $(MicrosoftAspNetCoreComponentsWebAssemblyServerPackageVersion) + $(MicrosoftAspNetCoreComponentsWebViewPackageVersion) + $(MicrosoftAspNetCoreDeveloperCertificatesXPlatPackageVersion) + $(MicrosoftAspNetCoreMetadataPackageVersion) + $(MicrosoftAspNetCoreMvcAnalyzersPackageVersion) + $(MicrosoftAspNetCoreMvcApiAnalyzersPackageVersion) + $(MicrosoftAspNetCoreMvcRazorExtensionsToolingInternalPackageVersion) + $(MicrosoftAspNetCoreTestHostPackageVersion) + $(MicrosoftBclAsyncInterfacesPackageVersion) + $(MicrosoftBuildPackageVersion) + $(MicrosoftBuildLocalizationPackageVersion) + $(MicrosoftBuildNuGetSdkResolverPackageVersion) + $(MicrosoftBuildTasksGitPackageVersion) + $(MicrosoftCodeAnalysisPackageVersion) + $(MicrosoftCodeAnalysisBuildClientPackageVersion) + $(MicrosoftCodeAnalysisCSharpPackageVersion) + $(MicrosoftCodeAnalysisCSharpCodeStylePackageVersion) + $(MicrosoftCodeAnalysisCSharpFeaturesPackageVersion) + $(MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion) + $(MicrosoftCodeAnalysisPublicApiAnalyzersPackageVersion) + $(MicrosoftCodeAnalysisRazorToolingInternalPackageVersion) + $(MicrosoftCodeAnalysisWorkspacesCommonPackageVersion) + $(MicrosoftCodeAnalysisWorkspacesMSBuildPackageVersion) + $(MicrosoftDeploymentDotNetReleasesPackageVersion) + $(MicrosoftDiaSymReaderPackageVersion) + $(MicrosoftDotNetArcadeSdkPackageVersion) + $(MicrosoftDotNetBuildTasksInstallersPackageVersion) + $(MicrosoftDotNetBuildTasksTemplatingPackageVersion) + $(MicrosoftDotNetBuildTasksWorkloadsPackageVersion) + $(MicrosoftDotNetHelixSdkPackageVersion) + $(MicrosoftDotNetSignToolPackageVersion) + $(MicrosoftDotNetWebItemTemplates100PackageVersion) + $(MicrosoftDotNetWebProjectTemplates100PackageVersion) + $(MicrosoftDotnetWinFormsProjectTemplatesPackageVersion) + $(MicrosoftDotNetWpfProjectTemplatesPackageVersion) + $(MicrosoftDotNetXliffTasksPackageVersion) + $(MicrosoftDotNetXUnitExtensionsPackageVersion) + $(MicrosoftExtensionsConfigurationIniPackageVersion) + $(MicrosoftExtensionsDependencyModelPackageVersion) + $(MicrosoftExtensionsFileProvidersAbstractionsPackageVersion) + $(MicrosoftExtensionsFileProvidersEmbeddedPackageVersion) + $(MicrosoftExtensionsFileSystemGlobbingPackageVersion) + $(MicrosoftExtensionsLoggingPackageVersion) + $(MicrosoftExtensionsLoggingAbstractionsPackageVersion) + $(MicrosoftExtensionsLoggingConsolePackageVersion) + $(MicrosoftExtensionsObjectPoolPackageVersion) + $(MicrosoftFSharpCompilerPackageVersion) + $(MicrosoftJSInteropPackageVersion) + $(MicrosoftMcpServerProjectTemplates100PackageVersion) + $(MicrosoftNetCompilersToolsetPackageVersion) + $(MicrosoftNetCompilersToolsetFrameworkPackageVersion) + $(MicrosoftNETHostModelPackageVersion) + $(MicrosoftNETILLinkTasksPackageVersion) + $(MicrosoftNETRuntimeEmscripten3156Cachewinx64PackageVersion) + $(MicrosoftNETRuntimeEmscriptenSdkInternalPackageVersion) + $(MicrosoftNETSdkRazorSourceGeneratorsTransportPackageVersion) + $(MicrosoftNETSdkWindowsDesktopPackageVersion) + $(MicrosoftNETTestSdkPackageVersion) + $(MicrosoftNETCoreAppRefPackageVersion) + $(MicrosoftNETCorePlatformsPackageVersion) + $(MicrosoftSourceLinkAzureReposGitPackageVersion) + $(MicrosoftSourceLinkBitbucketGitPackageVersion) + $(MicrosoftSourceLinkCommonPackageVersion) + $(MicrosoftSourceLinkGitHubPackageVersion) + $(MicrosoftSourceLinkGitLabPackageVersion) + $(MicrosoftTemplateEngineAbstractionsPackageVersion) + $(MicrosoftTemplateEngineAuthoringTemplateVerifierPackageVersion) + $(MicrosoftTemplateEngineEdgePackageVersion) + $(MicrosoftTemplateEngineMocksPackageVersion) + $(MicrosoftTemplateEngineOrchestratorRunnableProjectsPackageVersion) + $(MicrosoftTemplateEngineTestHelperPackageVersion) + $(MicrosoftTemplateEngineUtilsPackageVersion) + $(MicrosoftTemplateSearchCommonPackageVersion) + $(MicrosoftTemplateSearchTemplateDiscoveryPackageVersion) + $(MicrosoftTestPlatformBuildPackageVersion) + $(MicrosoftTestPlatformCLIPackageVersion) + $(MicrosoftWebXdtPackageVersion) + $(MicrosoftWin32SystemEventsPackageVersion) + $(MicrosoftWindowsDesktopAppInternalPackageVersion) + $(MicrosoftWindowsDesktopAppRefPackageVersion) + $(NuGetBuildTasksPackageVersion) + $(NuGetBuildTasksConsolePackageVersion) + $(NuGetBuildTasksPackPackageVersion) + $(NuGetCommandLineXPlatPackageVersion) + $(NuGetCommandsPackageVersion) + $(NuGetCommonPackageVersion) + $(NuGetConfigurationPackageVersion) + $(NuGetCredentialsPackageVersion) + $(NuGetDependencyResolverCorePackageVersion) + $(NuGetFrameworksPackageVersion) + $(NuGetLibraryModelPackageVersion) + $(NuGetLocalizationPackageVersion) + $(NuGetPackagingPackageVersion) + $(NuGetProjectModelPackageVersion) + $(NuGetProtocolPackageVersion) + $(NuGetVersioningPackageVersion) + $(SystemCodeDomPackageVersion) + $(SystemCommandLinePackageVersion) + $(SystemComponentModelCompositionPackageVersion) + $(SystemCompositionAttributedModelPackageVersion) + $(SystemCompositionConventionPackageVersion) + $(SystemCompositionHostingPackageVersion) + $(SystemCompositionRuntimePackageVersion) + $(SystemCompositionTypedPartsPackageVersion) + $(SystemConfigurationConfigurationManagerPackageVersion) + $(SystemDiagnosticsDiagnosticSourcePackageVersion) + $(SystemFormatsAsn1PackageVersion) + $(SystemIOHashingPackageVersion) + $(SystemReflectionMetadataLoadContextPackageVersion) + $(SystemResourcesExtensionsPackageVersion) + $(SystemSecurityCryptographyPkcsPackageVersion) + $(SystemSecurityCryptographyProtectedDataPackageVersion) + $(SystemSecurityCryptographyXmlPackageVersion) + $(SystemSecurityPermissionsPackageVersion) + $(SystemServiceProcessServiceControllerPackageVersion) + $(SystemTextEncodingCodePagesPackageVersion) + $(SystemTextJsonPackageVersion) + $(SystemWindowsExtensionsPackageVersion) + + $(MicrosoftTestingPlatformPackageVersion) + $(MSTestPackageVersion) + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e40900e9ac88..ab66dd096dd2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,634 +1,581 @@ + - - https://github.com/dotnet/templating - 1ad30c339011d797841a203de0a27c5106bad25f - - - https://github.com/dotnet/templating - 1ad30c339011d797841a203de0a27c5106bad25f - - - - https://github.com/dotnet/templating - b73682307aa0128c5edbec94c2e6a070d13ae6bb - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - - https://github.com/dotnet/emsdk - b634e009d59f72e9254f984a6b89e685955e0eb8 - - - - https://github.com/dotnet/emsdk - b634e009d59f72e9254f984a6b89e685955e0eb8 - - - - https://github.com/dotnet/msbuild - 2a0eb78b31025538b6e749ecd80dfabfcfa43a42 - - - https://github.com/dotnet/msbuild - 2a0eb78b31025538b6e749ecd80dfabfcfa43a42 - - - - https://github.com/dotnet/msbuild - 2a0eb78b31025538b6e749ecd80dfabfcfa43a42 - - - - https://github.com/dotnet/fsharp - 14987c804f33917bf15f4c25e0cd16ecd01807f4 - - - - https://github.com/dotnet/fsharp - 14987c804f33917bf15f4c25e0cd16ecd01807f4 - - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 - - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 6a953e76162f3f079405f80e28664fa51b136740 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/devdiv/DevDiv/_git/NuGet-NuGet.Client-Trusted - 13eeb32d62833984c9367f8f4e22a120ebc03b47 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://github.com/microsoft/vstest - 51441adcd6c424ae7315d66ce7e96baf34d70369 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://github.com/microsoft/vstest - 51441adcd6c424ae7315d66ce7e96baf34d70369 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://github.com/microsoft/vstest - 51441adcd6c424ae7315d66ce7e96baf34d70369 - - - - https://github.com/microsoft/vstest - 51441adcd6c424ae7315d66ce7e96baf34d70369 - + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - - - https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - - - https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 9c74c8784a90d64bcc5c5c552d5420eafb60fa4f - - - https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - a865df6e83bf8b34bbd6802fdcf25c5cdfb05191 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - - https://github.com/dotnet/razor - 41f3afd466695ac2460260431537fe4d779ff446 - - - https://github.com/dotnet/razor - 41f3afd466695ac2460260431537fe4d779ff446 - - - https://github.com/dotnet/razor - 41f3afd466695ac2460260431537fe4d779ff446 - - - - https://github.com/dotnet/razor - 41f3afd466695ac2460260431537fe4d779ff446 - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - 4569ad0b3bd84191660c74228ad6d18d29fbc1cb - - - https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - a865df6e83bf8b34bbd6802fdcf25c5cdfb05191 - - - https://github.com/dotnet/xdt - 63ae81154c50a1cf9287cc47d8351d55b4289e6d - - - - https://github.com/dotnet/xdt - 63ae81154c50a1cf9287cc47d8351d55b4289e6d - - - - https://github.com/dotnet/roslyn-analyzers - 94e92044351f9eb9e75b22f6000647393e36f5b9 - - - https://github.com/dotnet/roslyn - 450493a9b4ec6337bced0120e97cb76f4ed783db - - - - https://github.com/dotnet/roslyn-analyzers - 94e92044351f9eb9e75b22f6000647393e36f5b9 - - - - https://github.com/dotnet/command-line-api - 803d8598f98fb4efd94604b32627ee9407f246db - - - https://github.com/dotnet/command-line-api - 803d8598f98fb4efd94604b32627ee9407f246db + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - - https://github.com/dotnet/symreader - 0710a7892d89999956e8808c28e9dd0512bd53f3 - - - - https://github.com/dotnet/command-line-api - 803d8598f98fb4efd94604b32627ee9407f246db - - - - - https://github.com/dotnet/source-build-externals - 71dbdccd13f28cfd1a35649263b55ebbeab26ee7 - - - - - https://github.com/dotnet/source-build-reference-packages - 6092b62b7f35fddbd6bf31e19b2ab64bbe2443ae - - - - https://github.com/dotnet/deployment-tools - b2d5c0c5841de4bc036ef4c84b5db3532504e5f3 - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - - https://github.com/dotnet/sourcelink - 657ade4711e607cc4759e89e0943aa1ca8aadc63 - - - - - https://github.com/dotnet/deployment-tools - b2d5c0c5841de4bc036ef4c84b5db3532504e5f3 - - - - - https://github.com/dotnet/symreader - 0710a7892d89999956e8808c28e9dd0512bd53f3 - + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 5edf41b9c09952f18e404ad38e25670991c1b513 + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - - https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 - - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f2c8152eed158e72950025393fde498c90a57a6b - - - https://github.com/dotnet/arcade-services - e156e649f28395d9d0ee1e848225a689b59e0fd3 - - - https://github.com/dotnet/arcade-services - e156e649f28395d9d0ee1e848225a689b59e0fd3 - - - https://github.com/dotnet/scenario-tests - 623d5a21e18d6d1f370b5b9d1ebaf8d687464c0b - - - - https://github.com/dotnet/scenario-tests - 623d5a21e18d6d1f370b5b9d1ebaf8d687464c0b - - - - - https://github.com/dotnet/aspire - 5fa9337a84a52e9bd185d04d156eccbdcf592f74 - - - - https://github.com/dotnet/aspire - 5fa9337a84a52e9bd185d04d156eccbdcf592f74 - - - - https://github.com/dotnet/runtime - e77011b31a3e5c47d931248a64b47f9b2d47853d + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 + + + https://github.com/microsoft/testfx + 1ddd2f1a558e9c79b5327c5ccc0e9e89df39d4da + + + https://github.com/microsoft/testfx + 1ddd2f1a558e9c79b5327c5ccc0e9e89df39d4da + + + https://github.com/dotnet/dotnet + 09b877029dcd0b265a5425132ce4b03d8e7d7060 diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd index 56c2f25ac22f..ac1f72bf94e0 100644 --- a/eng/common/CIBuild.cmd +++ b/eng/common/CIBuild.cmd @@ -1,2 +1,2 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" \ No newline at end of file +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 792b60b49d42..65ed3a8adef0 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,13 +1,14 @@ # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables -# disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, +# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. +# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # # See example call for this script below. # # - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials +# displayName: Setup internal Feeds Credentials # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 @@ -34,19 +35,28 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 +# Adds or enables the package source with the given name +function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { + if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) { + AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password + } +} + # Add source entry to PackageSources function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) { + Write-Host "Adding package source $SourceName" + $packageSource = $doc.CreateElement("add") $packageSource.SetAttribute("key", $SourceName) $packageSource.SetAttribute("value", $SourceEndPoint) $sources.AppendChild($packageSource) | Out-Null } else { - Write-Host "Package source $SourceName already present." + Write-Host "Package source $SourceName already present and enabled." } AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd @@ -59,6 +69,8 @@ function AddCredential($creds, $source, $username, $pwd) { return; } + Write-Host "Inserting credential for feed: " $source + # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -91,24 +103,27 @@ function AddCredential($creds, $source, $username, $pwd) { $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd +# Enable all darc-int package sources. +function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) { + $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroInternalSources) { + EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key } } -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" +# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise. +function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) { + $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']") + if ($DisabledPackageSource) { + Write-Host "Enabling internal source '$($DisabledPackageSource.key)'." + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) + + AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password + return $true } + return $false } if (!(Test-Path $ConfigFile -PathType Leaf)) { @@ -121,15 +136,17 @@ $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName $doc.Load($filename) -# Get reference to or create one if none exist already +# Get reference to - fail if none exist $sources = $doc.DocumentElement.SelectSingleNode("packageSources") if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 } $creds = $null +$feedSuffix = "v3/index.json" if ($Password) { + $feedSuffix = "v2" # Looks for a node. Create it if none is found. $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") if ($creds -eq $null) { @@ -138,34 +155,35 @@ if ($Password) { } } +$userName = "dn-bot" + # Check for disabledPackageSources; we'll enable any darc-int ones we find there $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources + EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds } - -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password - -# 3.1 uses a different feed url format so it's handled differently here -$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password -} - -$dotnetVersions = @('5','6','7','8','9') +$dotnetVersions = @('5','6','7','8','9','10') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password } } +# Check for dotnet-eng and add dotnet-eng-internal if present +$dotnetEngSource = $sources.SelectSingleNode("add[@key='dotnet-eng']") +if ($dotnetEngSource -ne $null) { + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-eng-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password +} + +# Check for dotnet-tools and add dotnet-tools-internal if present +$dotnetToolsSource = $sources.SelectSingleNode("add[@key='dotnet-tools']") +if ($dotnetToolsSource -ne $null) { + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-tools-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password +} + $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh old mode 100644 new mode 100755 index facb415ca6ff..b2163abbe71b --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables -# disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, +# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. +# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -52,81 +53,139 @@ if [[ `uname -s` == "Darwin" ]]; then TB='' fi -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - ConfigNodeHeader="" - PackageSourcesTemplate="${TB}${NL}${TB}" +# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found. +EnableInternalPackageSource() { + local PackageSourceName="$1" + + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return 1 # No disabled sources section + fi + + # Check if this source name is disabled + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Enabling internal source '$PackageSourceName'." + # Remove the disabled entry (including any surrounding comments or whitespace on the same line) + sed -i.bak "//d" "$ConfigFile" + + # Add the source name to PackageSources for credential handling + PackageSources+=("$PackageSourceName") + return 0 # Found and enabled + fi + + return 1 # Not found in disabled sources +} + +# Add source entry to PackageSources +AddPackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Check if source already exists + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Package source $SourceName already present and enabled." + PackageSources+=("$SourceName") + return + fi + + echo "Adding package source $SourceName" + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile" + PackageSources+=("$SourceName") +} + +# Adds or enables the package source with the given name +AddOrEnablePackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Try to enable if disabled, if not found then add new source + EnableInternalPackageSource "$SourceName" + if [ "$?" != "0" ]; then + AddPackageSource "$SourceName" "$SourceEndPoint" + fi +} - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile -fi +# Enable all darc-int package sources +EnableMaestroInternalPackageSources() { + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return # No disabled sources section + fi + + # Find all darc-int disabled sources + local DisabledDarcIntSources=() + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"') + + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]]; then + EnableInternalPackageSource "$DisabledSourceName" + fi + done +} -# Ensure there is a ... section. -grep -i "" $ConfigFile +# Ensure there is a ... section. +grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding ... section." - - PackageSourcesNodeFooter="" - PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 fi PackageSources=() -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "... section. + grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + echo "Adding ... section." - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal-transport') fi -DotNetVersions=('5' '6' '7' '8' '9') +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +grep -i "" $ConfigFile > /dev/null +if [ "$?" == "0" ]; then + echo "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnableMaestroInternalPackageSources +fi + +DotNetVersions=('5' '6' '7' '8' '9' '10') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; - grep -i " /dev/null if [ "$?" == "0" ]; then - grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal") - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding $FeedPrefix-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal-transport") + AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix" + AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix" fi done +# Check for dotnet-eng and add dotnet-eng-internal if present +grep -i " /dev/null +if [ "$?" == "0" ]; then + AddOrEnablePackageSource "dotnet-eng-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$FeedSuffix" +fi + +# Check for dotnet-tools and add dotnet-tools-internal if present +grep -i " /dev/null +if [ "$?" == "0" ]; then + AddOrEnablePackageSource "dotnet-tools-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$FeedSuffix" +fi + # I want things split line by line PrevIFS=$IFS IFS=$'\n' @@ -139,29 +198,12 @@ if [ "$CredToken" ]; then # Check if there is no existing credential for this FeedName grep -i "<$FeedName>" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." + echo " Inserting credential for feed: $FeedName" PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + NewCredential="${TB}${TB}<$FeedName>${NL}${TB}${NL}${TB}${TB}${NL}${TB}${TB}" sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile fi done fi - -# Re-enable any entries in disabledPackageSources where the feed name contains darc-int -grep -i "" $ConfigFile -if [ "$?" == "0" ]; then - DisabledDarcIntSources=() - echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" - DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') - for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do - if [[ $DisabledSourceName == darc-int* ]] - then - OldDisableValue="" - NewDisableValue="" - sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile - echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" - fi - done -fi diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 438f9920c43e..8cfee107e7a3 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -7,6 +7,7 @@ Param( [string] $msbuildEngine = $null, [bool] $warnAsError = $true, [bool] $nodeReuse = $true, + [switch] $buildCheck = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, [switch][Alias('b')]$build, @@ -20,6 +21,7 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('pb')]$productBuild, + [switch]$fromVMR, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -71,6 +73,9 @@ function Print-Usage() { Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" + Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + Write-Host " -buildCheck Sets /check msbuild parameter" + Write-Host " -fromVMR Set when building from within the VMR" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -97,6 +102,7 @@ function Build { $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' } + $check = if ($buildCheck) { '/check' } else { '' } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -113,6 +119,7 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` $platformArg ` + $check ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` @@ -122,11 +129,13 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` - /p:DotNetBuildRepo=$productBuild ` + /p:DotNetBuild=$productBuild ` + /p:DotNetBuildFromVMR=$fromVMR ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` /p:Publish=$publish ` + /p:RestoreStaticGraphEnableBinaryLogger=$binaryLog ` @properties } diff --git a/eng/common/build.sh b/eng/common/build.sh index ac1ee8620cd2..9767bb411a4f 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -42,6 +42,8 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo " --buildCheck Sets /check msbuild parameter" + echo " --fromVMR Set when building from within the VMR" echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -63,6 +65,7 @@ restore=false build=false source_build=false product_build=false +from_vmr=false rebuild=false test=false integration_test=false @@ -76,6 +79,7 @@ clean=false warn_as_error=true node_reuse=true +build_check=false binary_log=false exclude_ci_binary_log=false pipelines_log=false @@ -87,7 +91,7 @@ verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' -properties='' +properties=() while [[ $# > 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in @@ -127,19 +131,22 @@ while [[ $# > 0 ]]; do -pack) pack=true ;; - -sourcebuild|-sb) + -sourcebuild|-source-build|-sb) build=true source_build=true product_build=true restore=true pack=true ;; - -productBuild|-pb) + -productbuild|-product-build|-pb) build=true product_build=true restore=true pack=true ;; + -fromvmr|-from-vmr) + from_vmr=true + ;; -test|-t) test=true ;; @@ -173,6 +180,9 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; + -buildcheck) + build_check=true + ;; -runtimesourcefeed) runtime_source_feed=$2 shift @@ -182,7 +192,7 @@ while [[ $# > 0 ]]; do shift ;; *) - properties="$properties $1" + properties+=("$1") ;; esac @@ -216,7 +226,7 @@ function Build { InitializeCustomToolset if [[ ! -z "$projects" ]]; then - properties="$properties /p:Projects=$projects" + properties+=("/p:Projects=$projects") fi local bl="" @@ -224,15 +234,21 @@ function Build { bl="/bl:\"$log_dir/Build.binlog\"" fi + local check="" + if [[ "$build_check" == true ]]; then + check="/check" + fi + MSBuild $_InitializeToolset \ $bl \ + $check \ /p:Configuration=$configuration \ /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ - /p:DotNetBuildRepo=$product_build \ - /p:ArcadeBuildFromSource=$source_build \ + /p:DotNetBuild=$product_build \ /p:DotNetBuildSourceOnly=$source_build \ + /p:DotNetBuildFromVMR=$from_vmr \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ @@ -240,7 +256,8 @@ function Build { /p:PerformanceTest=$performance_test \ /p:Sign=$sign \ /p:Publish=$publish \ - $properties + /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh index 1a02c0dec8fd..66e3b0ac61c3 100755 --- a/eng/common/cibuild.sh +++ b/eng/common/cibuild.sh @@ -13,4 +13,4 @@ while [[ -h $source ]]; do done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ \ No newline at end of file +. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 8da43d3b5837..eaed6d87e653 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -19,17 +19,17 @@ parameters: # publishing defaults artifacts: '' enableMicrobuild: false + enableMicrobuildForMacAndLinux: false microbuildUseESRP: true enablePublishBuildArtifacts: false enablePublishBuildAssets: false enablePublishTestResults: false - enablePublishUsingPipelines: false + enablePublishing: false enableBuildRetry: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' name: '' - componentGovernanceSteps: [] preSteps: [] artifactPublishSteps: [] runAsPublic: false @@ -74,9 +74,6 @@ jobs: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' # Retry signature validation up to three times, waiting 2 seconds between attempts. # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY @@ -128,23 +125,12 @@ jobs: - ${{ preStep }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(parameters.microbuildUseESRP, true) }}: - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea - ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + - template: /eng/common/core-templates/steps/install-microbuild.yml + parameters: + enableMicrobuild: ${{ parameters.enableMicrobuild }} + enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@1 @@ -160,27 +146,12 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} - richNavLogOutputDirectory: $(System.DefaultWorkingDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - ${{ each step in parameters.componentGovernanceSteps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/core-templates/steps/cleanup-microbuild.yml + parameters: + enableMicrobuild: ${{ parameters.enableMicrobuild }} + enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) # Publish test results - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index edefa789d360..eefed3b667a4 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -4,7 +4,7 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: '' - + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) @@ -27,7 +27,7 @@ parameters: is1ESPipeline: '' jobs: - job: OneLocBuild${{ parameters.JobNameSuffix }} - + dependsOn: ${{ parameters.dependsOn }} displayName: OneLocBuild${{ parameters.JobNameSuffix }} @@ -52,13 +52,13 @@ jobs: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows steps: @@ -86,8 +86,7 @@ jobs: isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} ${{ if eq(parameters.CreatePr, true) }}: isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} + isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: @@ -100,22 +99,20 @@ jobs: mirrorBranch: ${{ parameters.MirrorBranch }} condition: ${{ parameters.condition }} - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish Localization Files - pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' - publishLocation: Container - artifactName: Loc - condition: ${{ parameters.condition }} + # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact + - task: CopyFiles@2 + displayName: Copy LocProject.json + inputs: + SourceFolder: '$(System.DefaultWorkingDirectory)/eng/Localize/' + Contents: 'LocProject.json' + TargetFolder: '$(Build.ArtifactStagingDirectory)/loc' + condition: ${{ parameters.condition }} - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: - displayName: Publish LocProject.json - pathToPublish: '$(System.DefaultWorkingDirectory)/eng/Localize/' - publishLocation: Container - artifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + targetPath: '$(Build.ArtifactStagingDirectory)/loc' + artifactName: 'Loc' + displayName: 'Publish Localization Files' + condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 3cb20fb5041f..06f2eed0323d 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -20,9 +20,6 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing publishAssetsImmediately: false @@ -32,6 +29,15 @@ parameters: is1ESPipeline: '' + # Optional: 🌤️ or not the build has assets it wants to publish to BAR + isAssetlessBuild: false + + # Optional, publishing version + publishingVersion: 3 + + # Optional: A minimatch pattern for the asset manifests to publish to BAR + assetManifestsPattern: '*/manifests/**/*.xml' + repositoryAlias: self officialBuildId: '' @@ -68,13 +74,13 @@ jobs: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2022.amd64 + image: windows.vs2026.amd64 os: windows steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: @@ -84,18 +90,44 @@ jobs: - checkout: ${{ parameters.repositoryAlias }} fetchDepth: 3 clean: true - - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} + + - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: + - ${{ if eq(parameters.publishingVersion, 3) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Asset Manifests + inputs: + artifactName: AssetManifests + targetPath: '$(Build.StagingDirectory)/AssetManifests' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.publishingVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download V4 asset manifests + inputs: + itemPattern: '*/manifests/**/*.xml' + targetPath: '$(Build.StagingDirectory)/AllAssetManifests' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: CopyFiles@2 + displayName: Copy V4 asset manifests to AssetManifests + inputs: + SourceFolder: '$(Build.StagingDirectory)/AllAssetManifests' + Contents: ${{ parameters.assetManifestsPattern }} + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + flattenFolders: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} - task: NuGetAuthenticate@1 + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - task: AzureCLI@2 displayName: Publish Build Assets inputs: @@ -104,10 +136,13 @@ jobs: scriptLocation: scriptPath scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1 arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:ManifestsPath='$(Build.StagingDirectory)/AssetManifests' + /p:IsAssetlessBuild=${{ parameters.isAssetlessBuild }} /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(OfficialBuildId) + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} @@ -129,16 +164,28 @@ jobs: Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs" } - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - ${{ if eq(parameters.publishingVersion, 4) }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' + artifactName: AssetManifests + displayName: 'Publish Merged Manifest' + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing + + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish ReleaseConfigs Artifact - pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - publishLocation: Container + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' artifactName: ReleaseConfigs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} @@ -164,9 +211,13 @@ jobs: -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/core-templates/steps/publish-logs.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - JobLabel: 'Publish_Artifacts_Logs' + StageLabel: 'BuildAssetRegistry' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index d943748ac10e..1997c2ae00d7 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -12,9 +12,10 @@ parameters: # The name of the job. This is included in the job ID. # targetRID: '' # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false + # portableBuild: false # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. + # linux-x64), and compiling against distro-provided packages rather than portable ones. The + # default is portable mode. # skipPublishValidation: false # Disables publishing validation. By default, a check is performed to ensure no packages are # published by source-build. @@ -33,9 +34,6 @@ parameters: # container and pool. platform: {} - # Optional list of directories to ignore for component governance scans. - componentGovernanceIgnoreDirectories: [] - is1ESPipeline: '' # If set to true and running on a non-public project, @@ -62,7 +60,7 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2004.amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] image: build.azurelinux.3.amd64 @@ -71,10 +69,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} @@ -96,4 +94,3 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} platform: ${{ parameters.platform }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index ddf8c2e00d80..76baf5c27258 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -1,8 +1,5 @@ parameters: runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog @@ -16,12 +13,6 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexUploadPackageVersion - value: ${{ parameters.sourceIndexUploadPackageVersion }} - - name: SourceIndexProcessBinlogPackageVersion - value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - template: /eng/common/core-templates/variables/pool-providers.yml @@ -34,12 +25,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: 1es-windows-2022-open - os: windows + image: windows.vs2026preview.scout.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows + image: windows.vs2026preview.scout.amd64 steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: @@ -47,35 +36,9 @@ jobs: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - script: ${{ parameters.sourceIndexBuildCommand }} displayName: Build Repository - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - displayName: Log in to Azure and upload stage1 artifacts to source index - inputs: - azureSubscription: 'SourceDotNet Stage1 Publish' - addSpnToEnvironment: true - scriptType: 'ps' - scriptLocation: 'inlineScript' - inlineScript: | - $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml + parameters: + binLogPath: ${{ parameters.binLogPath }} diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml index 4571a7864df6..dbc14ac580a2 100644 --- a/eng/common/core-templates/jobs/codeql-build.yml +++ b/eng/common/core-templates/jobs/codeql-build.yml @@ -15,7 +15,6 @@ jobs: enablePublishBuildArtifacts: false enablePublishTestResults: false enablePublishBuildAssets: false - enablePublishUsingPipelines: false enableTelemetry: true variables: diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index bf33cdc2cc77..cc8cce452786 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -5,9 +5,6 @@ parameters: # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - # Optional: Enable running the source-build jobs to build repo from source enableSourceBuild: false @@ -30,6 +27,9 @@ parameters: # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. publishAssetsImmediately: false + # Optional: 🌤️ or not the build has assets it wants to publish to BAR + isAssetlessBuild: false + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' @@ -43,6 +43,10 @@ parameters: artifacts: {} is1ESPipeline: '' + + # Publishing version w/default. + publishingVersion: 3 + repositoryAlias: self officialBuildId: '' @@ -85,7 +89,6 @@ jobs: - template: /eng/common/core-templates/jobs/source-build.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - allCompletedJobId: Source_Build_Complete ${{ each parameter in parameters.sourceBuildParameters }}: ${{ parameter.key }}: ${{ parameter.value }} @@ -98,11 +101,12 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, ''), eq(parameters.isAssetlessBuild, true)) }}: - template: ../job/publish-build-assets.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} continueOnError: ${{ parameters.continueOnError }} + publishingVersion: ${{ parameters.publishingVersion }} dependsOn: - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.publishBuildAssetsDependsOn }}: @@ -110,12 +114,10 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + publishAssetsImmediately: ${{ or(parameters.publishAssetsImmediately, parameters.isAssetlessBuild) }} + isAssetlessBuild: ${{ parameters.isAssetlessBuild }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml index 0b408a67bd51..d92860cba208 100644 --- a/eng/common/core-templates/jobs/source-build.yml +++ b/eng/common/core-templates/jobs/source-build.yml @@ -2,28 +2,19 @@ parameters: # This template adds arcade-powered source-build to CI. A job is created for each platform, as # well as an optional server job that completes when all platform jobs complete. - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - # See /eng/common/core-templates/job/source-build.yml jobNamePrefix: 'Source_Build' # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-10-amd64' # Defines the platforms on which to run build jobs. One job is created for each platform, and the # object in this array is sent to the job template as 'platform'. If no platforms are specified, # one job runs on 'defaultManagedPlatform'. platforms: [] - # Optional list of directories to ignore for component governance scans. - componentGovernanceIgnoreDirectories: [] - is1ESPipeline: '' # If set to true and running on a non-public project, @@ -34,23 +25,12 @@ parameters: jobs: -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - - ${{ each platform in parameters.platforms }}: - template: /eng/common/core-templates/job/source-build.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: @@ -59,5 +39,4 @@ jobs: is1ESPipeline: ${{ parameters.is1ESPipeline }} jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 864427d9694a..905a6315e2d5 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -9,6 +9,7 @@ parameters: default: 3 values: - 3 + - 4 - name: BARBuildId displayName: BAR Build Id @@ -60,6 +61,11 @@ parameters: artifactNames: '' downloadArtifacts: true + - name: isAssetlessBuild + type: boolean + displayName: Is Assetless Build + default: false + # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation - name: symbolPublishingAdditionalParameters @@ -115,18 +121,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 + image: windows.vs2026preview.scout.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -135,16 +141,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true - task: PowerShell@2 displayName: Validate @@ -159,18 +179,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -178,19 +198,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -223,18 +254,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -242,16 +273,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/assets/**' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten assets to BlobArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/BlobArtifacts' + flattenFolders: true - task: PowerShell@2 displayName: Validate @@ -284,18 +329,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2022.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -305,6 +350,13 @@ stages: - task: NuGetAuthenticate@1 + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + # Darc is targeting 8.0, so make sure it's installed - task: UseDotNet@2 inputs: @@ -319,9 +371,12 @@ stages: scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: > -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -RequireDefaultChannels ${{ parameters.requireDefaultChannels }} -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index a7abd58c4bb6..6dfa99ec5e37 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -8,12 +8,11 @@ steps: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download Release Configs inputs: - buildType: current artifactName: ReleaseConfigs - checkDownloadedFiles: true + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' - task: AzureCLI@2 name: setReleaseVars diff --git a/eng/common/core-templates/steps/cleanup-microbuild.yml b/eng/common/core-templates/steps/cleanup-microbuild.yml new file mode 100644 index 000000000000..c0fdcd3379d7 --- /dev/null +++ b/eng/common/core-templates/steps/cleanup-microbuild.yml @@ -0,0 +1,28 @@ +parameters: + # Enable cleanup tasks for MicroBuild + enableMicrobuild: false + # Enable cleanup tasks for MicroBuild on Mac and Linux + # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' + enableMicrobuildForMacAndLinux: false + continueOnError: false + +steps: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and( + always(), + or( + and( + eq(variables['Agent.Os'], 'Windows_NT'), + in(variables['_SignType'], 'real', 'test') + ), + and( + ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }}, + ne(variables['Agent.Os'], 'Windows_NT'), + eq(variables['_SignType'], 'real') + ) + )) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml index 7f5b84c4cb82..aad0a8aeda33 100644 --- a/eng/common/core-templates/steps/generate-sbom.yml +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -1,54 +1,14 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - parameters: - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - is1ESPipeline: false - # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). - publishArtifacts: true + PackageVersion: unused + BuildDropPath: unused + PackageName: unused + ManifestDirPath: unused + IgnoreDirectories: unused + sbomContinueOnError: unused + is1ESPipeline: unused + publishArtifacts: unused steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- ${{ if eq(parameters.publishArtifacts, 'true')}}: - - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - targetPath: '${{ parameters.manifestDirPath }}' - artifactName: $(ARTIFACT_NAME) - + echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include." + displayName: Issue generate-sbom.yml deprecation warning diff --git a/eng/common/core-templates/steps/get-delegation-sas.yml b/eng/common/core-templates/steps/get-delegation-sas.yml index 9db5617ea7de..d2901470a7f0 100644 --- a/eng/common/core-templates/steps/get-delegation-sas.yml +++ b/eng/common/core-templates/steps/get-delegation-sas.yml @@ -31,16 +31,7 @@ steps: # Calculate the expiration of the SAS token and convert to UTC $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads - # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 - $sas = "" - do { - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - } while($sas.IndexOf('/') -ne -1) + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv if ($LASTEXITCODE -ne 0) { Write-Error "Failed to generate SAS token." diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml new file mode 100644 index 000000000000..553fce66b940 --- /dev/null +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -0,0 +1,110 @@ +parameters: + # Enable install tasks for MicroBuild + enableMicrobuild: false + # Enable install tasks for MicroBuild on Mac and Linux + # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' + enableMicrobuildForMacAndLinux: false + # Determines whether the ESRP service connection information should be passed to the signing plugin. + # This overlaps with _SignType to some degree. We only need the service connection for real signing. + # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place. + # Doing so will cause the service connection to be authorized for the pipeline, which isn't allowed and won't work for non-prod. + # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The + # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough. + microbuildUseESRP: true + # Microbuild installation directory + microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild + + continueOnError: false + +steps: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}: + # Needed to download the MicroBuild plugin nupkgs on Mac and Linux when nuget.exe is unavailable + - task: UseDotNet@2 + displayName: Install .NET 8.0 SDK for MicroBuild Plugin + inputs: + packageType: sdk + version: 8.0.x + installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + + - script: | + set -euo pipefail + + # UseDotNet@2 prepends the dotnet executable path to the PATH variable, so we can call dotnet directly + version=$(dotnet --version) + cat << 'EOF' > ${{ parameters.microBuildOutputFolder }}/global.json + { + "sdk": { + "version": "$version", + "paths": [ + "${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild" + ], + "errorMessage": "The .NET SDK version $version is required to install the MicroBuild signing plugin." + } + } + EOF + displayName: 'Add global.json to MicroBuild Installation path' + workingDirectory: ${{ parameters.microBuildOutputFolder }} + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + + - script: | + REM Check if ESRP is disabled while SignType is real + if /I "${{ parameters.microbuildUseESRP }}"=="false" if /I "$(_SignType)"=="real" ( + echo Error: ESRP must be enabled when SignType is real. + exit /b 1 + ) + displayName: 'Validate ESRP usage (Windows)' + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) + - script: | + # Check if ESRP is disabled while SignType is real + if [ "${{ parameters.microbuildUseESRP }}" = "false" ] && [ "$(_SignType)" = "real" ]; then + echo "Error: ESRP must be enabled when SignType is real." + exit 1 + fi + displayName: 'Validate ESRP usage (Non-Windows)' + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + + # Two different MB install steps. This is due to not being able to use the agent OS during + # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However, + # we can avoid including the MB install step if not enabled at all. This avoids a bunch of + # extra pipeline authorizations, since most pipelines do not sign on non-Windows. + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin (Windows) + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea + ${{ else }}: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) + + - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin (non-Windows) + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + workingDirectory: ${{ parameters.microBuildOutputFolder }} + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ${{ else }}: + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 0623ac6e1123..694f55a926e6 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -26,15 +26,15 @@ steps: # If the file exists - sensitive data for redaction will be sourced from it # (single entry per line, lines starting with '# ' are considered comments and skipped) arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' '$(dn-bot-all-orgs-build-rw-code-rw)' + '$(System.AccessToken)' ${{parameters.CustomSensitiveDataList}} continueOnError: true condition: always() @@ -45,14 +45,16 @@ steps: SourceFolder: '$(System.DefaultWorkingDirectory)/PostBuildLogs' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + condition: always() -- template: /eng/common/core-templates/steps/publish-build-artifacts.yml +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs + targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt) continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index 7846584d2a77..09ae5cd73ae3 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -11,10 +11,6 @@ parameters: # for details. The entire object is described in the 'job' template for simplicity, even though # the usage of the properties on this object is split between the 'job' and 'steps' templates. platform: {} - - # Optional list of directories to ignore for component governance scans. - componentGovernanceIgnoreDirectories: [] - is1ESPipeline: false steps: @@ -23,25 +19,12 @@ steps: set -x df -h - # If file changes are detected, set CopyWipIntoInnerSourceBuildRepo to copy the WIP changes into the inner source build repo. - internalRestoreArgs= - if ! git diff --quiet; then - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - # If building on the internal project, the internal storage variable may be available (usually only if needed) # In that case, add variables to allow the download of internal runtimes if the specified versions are not found # in the default public locations. internalRuntimeDownloadArgs= if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey '$(dotnetbuilds-internal-container-read-token-base64)'' fi buildConfig=Release @@ -50,88 +33,33 @@ steps: buildConfig='$(_BuildConfig)' fi - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + portableBuildArgs= + if [ '${{ parameters.platform.portableBuild }}' != '' ]; then + portableBuildArgs='/p:PortableBuild=${{ parameters.platform.portableBuild }}' fi ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ + --restore --build --pack -bl \ + --source-build \ ${{ parameters.platform.buildArguments }} \ - $officialBuildArgs \ $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ - /p:AssetManifestFileName=$assetManifestFileName + $portableBuildArgs \ displayName: Build -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(System.DefaultWorkingDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/sb/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish BuildLogs - targetPath: '$(Build.StagingDirectory)/BuildLogs' + targetPath: artifacts/log/${{ coalesce(variables._BuildConfig, 'Release') }} artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() - sbomEnabled: false # we don't need SBOM for logs - -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - displayName: Component Detection (Exclude upstream cache) - is1ESPipeline: ${{ parameters.is1ESPipeline }} - ${{ if eq(length(parameters.componentGovernanceIgnoreDirectories), 0) }}: - componentGovernanceIgnoreDirectories: '$(System.DefaultWorkingDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' - ${{ else }}: - componentGovernanceIgnoreDirectories: ${{ join(',', parameters.componentGovernanceIgnoreDirectories) }} - disableComponentGovernance: ${{ eq(variables['System.TeamProject'], 'public') }} + isProduction: false # logs are non-production artifacts diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml new file mode 100644 index 000000000000..6e7666b4dcf8 --- /dev/null +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -0,0 +1,35 @@ +parameters: + sourceIndexUploadPackageVersion: 2.0.0-20250818.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + binlogPath: artifacts/log/Debug/Build.binlog + +steps: +- task: UseDotNet@2 + displayName: "Source Index: Use .NET 9 SDK" + inputs: + packageType: sdk + version: 9.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + +- script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: "Source Index: Download netsourceindex Tools" + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + +- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: "Source Index: Process Binlog into indexable sln" + +- ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: AzureCLI@2 + displayName: "Source Index: Upload Source Index stage1 artifacts to Azure" + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch index af7c8be05906..2cebc547382e 100644 --- a/eng/common/cross/arm64/tizen/tizen.patch +++ b/eng/common/cross/arm64/tizen/tizen.patch @@ -5,5 +5,5 @@ diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so Use the shared library, but some functions are only in the static library, so try that secondarily. */ OUTPUT_FORMAT(elf64-littleaarch64) --GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) ) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-aarch64.so.1 ) ) +GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) ) diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 7e9ba2b75ed3..fbd8d80848a6 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -6,10 +6,11 @@ usage() { echo "Creates a toolchain and sysroot used for cross-compiling for Android." echo - echo "Usage: $0 [BuildArch] [ApiLevel]" + echo "Usage: $0 [BuildArch] [ApiLevel] [--ndk NDKVersion]" echo echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" + echo "NDKVersion is the version of Android NDK. The default is r21. See https://developer.android.com/ndk/downloads/revision_history" echo echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" echo "by setting the TOOLCHAIN_DIR environment variable" @@ -25,10 +26,15 @@ __BuildArch=arm64 __AndroidArch=aarch64 __AndroidToolchain=aarch64-linux-android -for i in "$@" - do - lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" - case $lowerI in +while :; do + if [[ "$#" -le 0 ]]; then + break + fi + + i=$1 + + lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" + case $lowerI in -?|-h|--help) usage exit 1 @@ -43,6 +49,10 @@ for i in "$@" __AndroidArch=arm __AndroidToolchain=arm-linux-androideabi ;; + --ndk) + shift + __NDK_Version=$1 + ;; *[0-9]) __ApiLevel=$i ;; @@ -50,8 +60,17 @@ for i in "$@" __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" ;; esac + shift done +if [[ "$__NDK_Version" == "r21" ]] || [[ "$__NDK_Version" == "r22" ]]; then + __NDK_File_Arch_Spec=-x86_64 + __SysRoot=sysroot +else + __NDK_File_Arch_Spec= + __SysRoot=toolchains/llvm/prebuilt/linux-x86_64/sysroot +fi + # Obtain the location of the bash script to figure out where the root of the repo is. __ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -78,6 +97,7 @@ fi echo "Target API level: $__ApiLevel" echo "Target architecture: $__BuildArch" +echo "NDK version: $__NDK_Version" echo "NDK location: $__NDK_Dir" echo "Target Toolchain location: $__ToolchainDir" @@ -85,8 +105,8 @@ echo "Target Toolchain location: $__ToolchainDir" if [ ! -d $__NDK_Dir ]; then echo Downloading the NDK into $__NDK_Dir mkdir -p $__NDK_Dir - wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux$__NDK_File_Arch_Spec.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux.zip + unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux.zip -d $__CrossDir fi if [ ! -d $__lldb_Dir ]; then @@ -116,16 +136,11 @@ for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/ fi done -cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" +cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/$__SysRoot/usr/" # Generate platform file for build.sh script to assign to __DistroRid echo "Generating platform file..." -echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform - -echo "Now to build coreclr, libraries and installers; run:" -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory coreclr -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory libraries -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory installer +echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/$__SysRoot/android_platform + +echo "Now to build coreclr, libraries and host; run:" +echo ROOTFS_DIR=$(realpath $__ToolchainDir/$__SysRoot) ./build.sh clr+libs+host --cross --arch $__BuildArch diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 4b5e8d7166bd..3150ccac6fcd 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -5,16 +5,18 @@ set -e usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" + echo "BuildArch can be: arm(default), arm64, loongarch64, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" echo " for FreeBSD can be: freebsd13, freebsd14" + echo " for OpenBSD can be: openbsd" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)." + echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." echo "--jobs N - optional, restrict to N jobs." exit 1 @@ -26,6 +28,8 @@ __BuildArch=arm __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 +__OpenBSDArch=arm +__OpenBSDMachineArch=armv7 __IllumosArch=arm7 __HaikuArch=arm __QEMUArch=arm @@ -52,28 +56,27 @@ __UbuntuPackages+=" symlinks" __UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" libunwind8-dev" -__UbuntuPackages+=" libnuma-dev" __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" __AlpinePackages+=" compiler-rt" -__AlpinePackages+=" numactl-dev" # runtime libraries' dependencies __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" zlib1g-dev" +__UbuntuPackages+=" libbrotli-dev" __AlpinePackages+=" curl-dev" __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.3-RELEASE" -__FreeBSDPkg="1.17.0" +__FreeBSDBase="13.5-RELEASE" +__FreeBSDPkg="2.7.5" __FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" @@ -82,6 +85,13 @@ __FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" +__OpenBSDVersion="7.8" +__OpenBSDPackages="heimdal-libs" +__OpenBSDPackages+=" icu4c" +__OpenBSDPackages+=" libinotify" +__OpenBSDPackages+=" openssl" +__OpenBSDPackages+=" e2fsprogs" + __IllumosPackages="icu" __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" @@ -91,18 +101,18 @@ __HaikuPackages="gcc_syslibs" __HaikuPackages+=" gcc_syslibs_devel" __HaikuPackages+=" gmp" __HaikuPackages+=" gmp_devel" -__HaikuPackages+=" icu66" -__HaikuPackages+=" icu66_devel" +__HaikuPackages+=" icu[0-9]+" +__HaikuPackages+=" icu[0-9]*_devel" __HaikuPackages+=" krb5" __HaikuPackages+=" krb5_devel" __HaikuPackages+=" libiconv" __HaikuPackages+=" libiconv_devel" -__HaikuPackages+=" llvm12_libunwind" -__HaikuPackages+=" llvm12_libunwind_devel" +__HaikuPackages+=" llvm[0-9]*_libunwind" +__HaikuPackages+=" llvm[0-9]*_libunwind_devel" __HaikuPackages+=" mpfr" __HaikuPackages+=" mpfr_devel" -__HaikuPackages+=" openssl" -__HaikuPackages+=" openssl_devel" +__HaikuPackages+=" openssl3" +__HaikuPackages+=" openssl3_devel" __HaikuPackages+=" zlib" __HaikuPackages+=" zlib_devel" @@ -128,10 +138,11 @@ __AlpineKeys=' 616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ== 616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ== 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== +66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ== ' -__Keyring= __KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 +__SkipEmulation=0 __UseMirror=0 __UnprocessedBuildArgs= @@ -151,6 +162,10 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armel) + # this is only used for tizen-build-rootfs.sh + __BuildArch=armel + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -158,27 +173,16 @@ while :; do __QEMUArch=aarch64 __FreeBSDArch=arm64 __FreeBSDMachineArch=aarch64 + __OpenBSDArch=arm64 + __OpenBSDMachineArch=aarch64 ;; - armel) - __BuildArch=armel - __UbuntuArch=armel - __UbuntuRepo="http://ftp.debian.org/debian/" - __CodeName=jessie - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" - ;; - armv6) - __BuildArch=armv6 - __UbuntuArch=armhf - __QEMUArch=arm - __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" - __CodeName=buster - __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg" - __LLDB_Package="liblldb-6.0-dev" - __UbuntuSuites= - - if [[ -e "$__KeyringFile" ]]; then - __Keyring="--keyring $__KeyringFile" - fi + loongarch64) + __BuildArch=loongarch64 + __AlpineArch=loongarch64 + __QEMUArch=loongarch64 + __UbuntuArch=loong64 + __UbuntuSuites=unreleased + __LLDB_Package="liblldb-19-dev" ;; riscv64) __BuildArch=riscv64 @@ -194,7 +198,7 @@ while :; do __AlpineArch=ppc64le __QEMUArch=ppc64le __UbuntuArch=ppc64el - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp5/}" @@ -205,7 +209,7 @@ while :; do __AlpineArch=s390x __QEMUArch=s390x __UbuntuArch=s390x - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp5/}" @@ -217,15 +221,17 @@ while :; do __UbuntuArch=amd64 __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 + __OpenBSDArch=amd64 + __OpenBSDMachineArch=amd64 __illumosArch=x86_64 __HaikuArch=x86_64 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + __UbuntuRepo="https://archive.ubuntu.com/ubuntu/" ;; x86) __BuildArch=x86 __UbuntuArch=i386 __AlpineArch=x86 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + __UbuntuRepo="https://archive.ubuntu.com/ubuntu/" ;; lldb*) version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" @@ -264,45 +270,20 @@ while :; do ;; xenial) # Ubuntu 16.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=xenial - fi - ;; - zesty) # Ubuntu 17.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=zesty - fi + __CodeName=xenial ;; bionic) # Ubuntu 18.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=bionic - fi + __CodeName=bionic ;; focal) # Ubuntu 20.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=focal - fi + __CodeName=focal ;; jammy) # Ubuntu 22.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=jammy - fi + __CodeName=jammy ;; noble) # Ubuntu 24.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=noble - fi - if [[ -n "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-18-dev" - fi - ;; - jessie) # Debian 8 - __CodeName=jessie - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi + __CodeName=noble + __LLDB_Package="liblldb-19-dev" ;; stretch) # Debian 9 __CodeName=stretch @@ -310,7 +291,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://archive.debian.org/debian/" fi ;; buster) # Debian 10 @@ -319,7 +300,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://archive.debian.org/debian/" fi ;; bullseye) # Debian 11 @@ -327,7 +308,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; bookworm) # Debian 12 @@ -335,16 +316,30 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; sid) # Debian sid __CodeName=sid - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" + __UbuntuSuites= - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi + # Debian-Ports architectures need different values + case "$__UbuntuArch" in + amd64|arm64|armhf|i386|mips64el|ppc64el|riscv64|s390x) + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="https://ftp.debian.org/debian/" + fi + ;; + *) + __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="https://ftp.debian.org/debian-ports/" + fi + ;; + esac ;; tizen) __CodeName= @@ -370,10 +365,14 @@ while :; do ;; freebsd14) __CodeName=freebsd - __FreeBSDBase="14.0-RELEASE" + __FreeBSDBase="14.3-RELEASE" __FreeBSDABI="14" __SkipUnmount=1 ;; + openbsd) + __CodeName=openbsd + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __SkipUnmount=1 @@ -388,6 +387,9 @@ while :; do --skipsigcheck) __SkipSigCheck=1 ;; + --skipemulation) + __SkipEmulation=1 + ;; --rootfsdir|-rootfsdir) shift __RootfsDir="$1" @@ -420,16 +422,15 @@ case "$__AlpineVersion" in elif [[ "$__AlpineArch" == "x86" ]]; then __AlpineVersion=3.17 # minimum version that supports lldb-dev __AlpinePackages+=" llvm15-libs" - elif [[ "$__AlpineArch" == "riscv64" ]]; then + elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then + __AlpineVersion=3.21 # minimum version that supports lldb-dev + __AlpinePackages+=" llvm19-libs" + elif [[ -n "$__AlpineMajorVersion" ]]; then + # use whichever alpine version is provided and select the latest toolchain libs __AlpineLlvmLibsLookup=1 - __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive) else __AlpineVersion=3.13 # 3.13 to maximize compatibility __AlpinePackages+=" llvm10-libs" - - if [[ "$__AlpineArch" == "armv7" ]]; then - __AlpinePackages="${__AlpinePackages//numactl-dev/}" - fi fi esac @@ -439,19 +440,10 @@ if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then __AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}" fi -if [[ "$__BuildArch" == "armel" ]]; then - __LLDB_Package="lldb-3.5-dev" -fi - -if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then - # libnuma-dev is not available on armhf for xenial - __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}" -fi - __UbuntuPackages+=" ${__LLDB_Package:-}" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ports.ubuntu.com/" + __UbuntuRepo="https://ports.ubuntu.com/" fi if [[ -n "$__LLVM_MajorVersion" ]]; then @@ -496,7 +488,7 @@ if [[ "$__CodeName" == "alpine" ]]; then arch="$(uname -m)" ensureDownloadTool - + if [[ "$__hasWget" == 1 ]]; then wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" else @@ -512,11 +504,6 @@ if [[ "$__CodeName" == "alpine" ]]; then echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c chmod +x "$__ApkToolsDir/apk.static" - if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then - mkdir -p "$__RootfsDir"/usr/bin - cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" - fi - if [[ "$__AlpineVersion" == "edge" ]]; then version=edge else @@ -536,18 +523,22 @@ if [[ "$__CodeName" == "alpine" ]]; then __ApkSignatureArg="--keys-dir $__ApkKeysDir" fi + if [[ "$__SkipEmulation" == "1" ]]; then + __NoEmulationArg="--no-scripts" + fi + # initialize DB # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then # shellcheck disable=SC2086 __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')" fi @@ -555,9 +546,9 @@ if [[ "$__CodeName" == "alpine" ]]; then # install all packages in one go # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \ add $__AlpinePackages rm -r "$__ApkToolsDir" @@ -573,7 +564,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version fi echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf - echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf + echo "FreeBSD: { url: \"pkg+https://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf mkdir -p "$__RootfsDir"/tmp # get and build package manager if [[ "$__hasWget" == 1 ]]; then @@ -587,9 +578,69 @@ elif [[ "$__CodeName" == "freebsd" ]]; then ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # install packages we need. - INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) IGNORE_OSVERSION=yes "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update # shellcheck disable=SC2086 INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "openbsd" ]]; then + # determine mirrors + OPENBSD_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/$__OpenBSDVersion/$__OpenBSDMachineArch" + + # download base system sets + ensureDownloadTool + + BASE_SETS=(base comp) + for set in "${BASE_SETS[@]}"; do + FILE="${set}${__OpenBSDVersion//./}.tgz" + echo "Downloading $FILE..." + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + PKG_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/${__OpenBSDVersion}/packages/${__OpenBSDMachineArch}" + + echo "Installing packages into sysroot..." + + # Fetch package index once + if [[ "$__hasWget" == 1 ]]; then + PKG_INDEX=$(wget -qO- "$PKG_MIRROR/") + else + PKG_INDEX=$(curl -s "$PKG_MIRROR/") + fi + + for pkg in $__OpenBSDPackages; do + PKG_FILE=$(echo "$PKG_INDEX" | grep -Po ">\K${pkg}-[0-9][^\" ]*\.tgz" \ + | sort -V | tail -n1) + + echo "Resolved package filename for $pkg: $PKG_FILE" + + [[ -z "$PKG_FILE" ]] && { echo "ERROR: Package $pkg not found"; exit 1; } + + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir/usr/local" -xzpf - + else + curl -SL "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir/usr/local" -xzpf - + fi + done + + echo "Creating versionless symlinks for shared libraries..." + # Find all versioned .so files and create the base .so symlink + for lib in "$__RootfsDir"/usr/lib/lib*.so.*; do + if [ -f "$lib" ]; then + # Extract the filename (e.g., libc++.so.12.0) + VERSIONED_NAME=$(basename "$lib") + # Remove the trailing version numbers (e.g., libc++.so) + BASE_NAME=${VERSIONED_NAME%.so.*}.so + # Create the symlink in the same directory + ln -sf "$VERSIONED_NAME" "$__RootfsDir/usr/lib/$BASE_NAME" + fi + done + + echo "Cleaning up unnecessary paths" + # we don't use executables and kernel in rootfs (as we use host's compiler with -sysroot) + rm -rf "$__RootfsDir/usr/share" "$__RootfsDir/usr/bin" elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" @@ -681,7 +732,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then ensureDownloadTool - echo "Downloading Haiku package tool" + echo "Downloading Haiku package tools" git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script" if [[ "$__hasWget" == 1 ]]; then wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" @@ -691,34 +742,42 @@ elif [[ "$__CodeName" == "haiku" ]]; then unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" - DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current" + + echo "Downloading HaikuPorts package repository index..." + if [[ "$__hasWget" == 1 ]]; then + wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo" + else + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo" + fi - # Download Haiku packages echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." - # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 - # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 + hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" | + grep -E "${package}-" | sort -V | tail -n 1 | xargs)" + if [ -z "$hpkgFilename" ]; then + >&2 echo "ERROR: package $package missing." + exit 1 + fi + echo "Resolved filename: $hpkgFilename..." + hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename" if [[ "$__hasWget" == 1 ]]; then - hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" else - hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" fi done for package in haiku haiku_devel; do echo "Downloading $package..." if [[ "$__hasWget" == 1 ]]; then - hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" else - hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" fi done @@ -744,25 +803,72 @@ elif [[ "$__CodeName" == "haiku" ]]; then popd rm -rf "$__RootfsDir/tmp" elif [[ -n "$__CodeName" ]]; then + __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)" + __SigCheckArgs= if [[ "$__SkipSigCheck" == "0" ]]; then - __Keyring="$__Keyring --force-check-gpg" + if [[ -e "$__KeyringFile" ]]; then + __SigCheckArgs="--keyring $__KeyringFile" + fi + __SigCheckArgs="$__SigCheckArgs --force-check-gpg" + fi + + if [[ "$__SkipEmulation" == "1" ]]; then + if [[ -z "$AR" ]]; then + if command -v ar &>/dev/null; then + AR="$(command -v ar)" + elif command -v llvm-ar &>/dev/null; then + AR="$(command -v llvm-ar)" + else + echo "Unable to find ar or llvm-ar on PATH, add them to PATH or set AR environment variable pointing to the available AR tool" + exit 1 + fi + fi + + PYTHON=${PYTHON_EXECUTABLE:-python3} + + # shellcheck disable=SC2086,SC2046 + echo running "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ + $__UbuntuPackages + + # shellcheck disable=SC2086,SC2046 + "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ + $__UbuntuPackages + + exit 0 fi # shellcheck disable=SC2086 - echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" - debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + echo running debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + # shellcheck disable=SC2086 + if ! debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then + echo "debootstrap failed! dumping debootstrap.log" + cat "$__RootfsDir/debootstrap/debootstrap.log" + exit 1 + fi + + rm -rf "$__RootfsDir"/etc/apt/*.{sources,list} "$__RootfsDir"/etc/apt/sources.list.d mkdir -p "$__RootfsDir/etc/apt/sources.list.d/" + + # shellcheck disable=SC2086 cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" < + matches = re.findall(r'^ (\S*) +(\S*) +(\S*)$', content, re.MULTILINE) + + for entry in matches: + # the file has both md5 and sha256 checksums, we want sha256 which has a length of 64 + if entry[2] == path and len(entry[0]) == 64: + return entry[0] + + raise Exception(f"Could not find checksum for {path} in Release file.") + +def parse_debian_version(version): + """Parse a Debian package version into epoch, upstream version, and revision.""" + match = re.match(r'^(?:(\d+):)?([^-]+)(?:-(.+))?$', version) + if not match: + raise ValueError(f"Invalid Debian version format: {version}") + epoch, upstream, revision = match.groups() + return int(epoch) if epoch else 0, upstream, revision or "" + +def compare_upstream_version(v1, v2): + """Compare upstream or revision parts using Debian rules.""" + def tokenize(version): + tokens = re.split(r'([0-9]+|[A-Za-z]+)', version) + return [int(x) if x.isdigit() else x for x in tokens if x] + + tokens1 = tokenize(v1) + tokens2 = tokenize(v2) + + for token1, token2 in zip(tokens1, tokens2): + if type(token1) == type(token2): + if token1 != token2: + return (token1 > token2) - (token1 < token2) + else: + return -1 if isinstance(token1, str) else 1 + + return len(tokens1) - len(tokens2) + +def compare_debian_versions(version1, version2): + """Compare two Debian package versions.""" + epoch1, upstream1, revision1 = parse_debian_version(version1) + epoch2, upstream2, revision2 = parse_debian_version(version2) + + if epoch1 != epoch2: + return epoch1 - epoch2 + + result = compare_upstream_version(upstream1, upstream2) + if result != 0: + return result + + return compare_upstream_version(revision1, revision2) + +def resolve_dependencies(packages, aliases, desired_packages): + """Recursively resolves dependencies for the desired packages.""" + resolved = [] + to_process = deque(desired_packages) + + while to_process: + current = to_process.popleft() + resolved_package = current if current in packages else aliases.get(current, [None])[0] + + if not resolved_package: + print(f"Error: Package '{current}' was not found in the available packages.") + sys.exit(1) + + if resolved_package not in resolved: + resolved.append(resolved_package) + + deps = packages.get(resolved_package, {}).get("Depends", "") + if deps: + deps = [dep.split(' ')[0] for dep in deps.split(', ') if dep] + for dep in deps: + if dep not in resolved and dep not in to_process and dep in packages: + to_process.append(dep) + + return resolved + +def parse_package_index(content): + """Parses the Packages.gz file and returns package information.""" + packages = {} + aliases = {} + entries = re.split(r'\n\n+', content) + + for entry in entries: + fields = dict(re.findall(r'^(\S+): (.+)$', entry, re.MULTILINE)) + if "Package" in fields: + package_name = fields["Package"] + version = fields.get("Version") + filename = fields.get("Filename") + depends = fields.get("Depends") + provides = fields.get("Provides", None) + sha256 = fields.get("SHA256") + + # Only update if package_name is not in packages or if the new version is higher + if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0: + packages[package_name] = { + "Version": version, + "Filename": filename, + "Depends": depends, + "SHA256": sha256 + } + + # Update aliases if package provides any alternatives + if provides: + provides_list = [x.strip() for x in provides.split(",")] + for alias in provides_list: + # Strip version specifiers + alias_name = re.sub(r'\s*\(=.*\)', '', alias) + if alias_name not in aliases: + aliases[alias_name] = [] + if package_name not in aliases[alias_name]: + aliases[alias_name].append(package_name) + + return packages, aliases + +def install_packages(mirror, packages_info, aliases, tmp_dir, extract_dir, ar_tool, desired_packages): + """Downloads .deb files and extracts them.""" + resolved_packages = resolve_dependencies(packages_info, aliases, desired_packages) + print(f"Resolved packages (including dependencies): {resolved_packages}") + + packages_to_download = {} + + for pkg in resolved_packages: + if pkg in packages_info: + packages_to_download[pkg] = packages_info[pkg] + + if pkg in aliases: + for alias in aliases[pkg]: + if alias in packages_info: + packages_to_download[alias] = packages_info[alias] + + asyncio.run(download_deb_files_parallel(mirror, packages_to_download, tmp_dir)) + + package_to_deb_file_map = {} + for pkg in resolved_packages: + pkg_info = packages_info.get(pkg) + if pkg_info: + deb_filename = pkg_info.get("Filename") + if deb_filename: + deb_file_path = os.path.join(tmp_dir, os.path.basename(deb_filename)) + package_to_deb_file_map[pkg] = deb_file_path + + for pkg in reversed(resolved_packages): + deb_file = package_to_deb_file_map.get(pkg) + if deb_file and os.path.exists(deb_file): + extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool) + + print("All done!") + +def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): + """Extract .deb file contents""" + + os.makedirs(extract_dir, exist_ok=True) + + with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir: + result = subprocess.run([ar_tool, "t", os.path.abspath(deb_file)], cwd=tmp_subdir, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + tar_filename = None + for line in result.stdout.decode().splitlines(): + if line.startswith("data.tar"): + tar_filename = line.strip() + break + + if not tar_filename: + raise FileNotFoundError(f"Could not find 'data.tar.*' in {deb_file}.") + + tar_file_path = os.path.join(tmp_subdir, tar_filename) + print(f"Extracting {tar_filename} from {deb_file}..") + + with open(tar_file_path, "wb") as outfile: + subprocess.run([ar_tool, "p", os.path.abspath(deb_file), tar_filename], check=True, stdout=outfile, stderr=subprocess.PIPE) + + file_extension = os.path.splitext(tar_file_path)[1].lower() + + if file_extension == ".xz": + mode = "r:xz" + elif file_extension == ".gz": + mode = "r:gz" + elif file_extension == ".zst": + # zstd is not supported by standard library yet + decompressed_tar_path = tar_file_path.replace(".zst", "") + with open(tar_file_path, "rb") as zst_file, open(decompressed_tar_path, "wb") as decompressed_file: + dctx = zstandard.ZstdDecompressor() + dctx.copy_stream(zst_file, decompressed_file) + + tar_file_path = decompressed_tar_path + mode = "r" + else: + raise ValueError(f"Unsupported compression format: {file_extension}") + + with tarfile.open(tar_file_path, mode) as tar: + tar.extractall(path=extract_dir, filter=_rootfs_extraction_filter) + +def _rootfs_extraction_filter(member, dest_path): + """Tarfile extraction filter based on the 'data' filter that additionally + rewrites absolute-target symlinks/hardlinks into rootfs-relative paths. + """ + if (member.issym() or member.islnk()) and os.path.isabs(member.linkname): + link_dir = os.path.dirname(member.name) + new_linkname = os.path.relpath(member.linkname.lstrip('/'), + start=link_dir or '.') + member = member.replace(linkname=new_linkname, deep=False) + return tarfile.data_filter(member, dest_path) + +def finalize_setup(rootfsdir): + lib_dir = os.path.join(rootfsdir, 'lib') + usr_lib_dir = os.path.join(rootfsdir, 'usr', 'lib') + + if os.path.exists(lib_dir): + if os.path.islink(lib_dir): + os.remove(lib_dir) + else: + os.makedirs(usr_lib_dir, exist_ok=True) + + for item in os.listdir(lib_dir): + src = os.path.join(lib_dir, item) + dest = os.path.join(usr_lib_dir, item) + + if os.path.isdir(src): + shutil.copytree(src, dest, dirs_exist_ok=True) + else: + shutil.copy2(src, dest) + + shutil.rmtree(lib_dir) + + os.symlink(usr_lib_dir, lib_dir) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS") + parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)") + parser.add_argument("--rootfsdir", required=True, help="Destination directory.") + parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.') + parser.add_argument("--mirror", required=True, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)") + parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)") + parser.add_argument("--force-check-gpg", required=False, action='store_true', help="Verify the packages against signatures in Release file.") + parser.add_argument("--keyring", required=False, default='', help="Keyring file to check signature of Release file.") + parser.add_argument("packages", nargs="+", help="List of package names to be installed.") + + args = parser.parse_args() + + DESIRED_PACKAGES = args.packages + [ # base packages + "dpkg", + "busybox", + "libc-bin", + "base-files", + "base-passwd", + "debianutils" + ] + + print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}") + + check_sig = args.force_check_gpg + if check_sig and not args.keyring: + print("ERROR: --force-check-gpg requires --keyring to specify a keyring file for signature verification.") + print("Install the appropriate keyring package (e.g., debian-ports-archive-keyring, ubuntu-archive-keyring)") + print("or pass --skipsigcheck to build-rootfs.sh to disable signature checking.") + sys.exit(1) + + package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite, check_sig, args.keyring)) + + packages_info, aliases = parse_package_index(package_index_content) + + with tempfile.TemporaryDirectory() as tmp_dir: + install_packages(args.mirror, packages_info, aliases, tmp_dir, args.rootfsdir, args.artool, DESIRED_PACKAGES) + + finalize_setup(args.rootfsdir) diff --git a/eng/common/cross/tizen-build-rootfs.sh b/eng/common/cross/tizen-build-rootfs.sh old mode 100644 new mode 100755 diff --git a/eng/common/cross/tizen-fetch.sh b/eng/common/cross/tizen-fetch.sh old mode 100644 new mode 100755 index 28936ceef3a7..37c3a61f1de8 --- a/eng/common/cross/tizen-fetch.sh +++ b/eng/common/cross/tizen-fetch.sh @@ -156,13 +156,8 @@ fetch_tizen_pkgs() done } -if [ "$TIZEN_ARCH" == "riscv64" ]; then - BASE="Tizen-Base-RISCV" - UNIFIED="Tizen-Unified-RISCV" -else - BASE="Tizen-Base" - UNIFIED="Tizen-Unified" -fi +BASE="Tizen-Base" +UNIFIED="Tizen-Unified" Inform "Initialize ${TIZEN_ARCH} base" fetch_tizen_pkgs_init standard $BASE diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 9a7ecfbd42c5..f65c689f6954 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,15 +3,22 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) # reset platform variables (e.g. cmake 3.25 sets LINUX=1) unset(LINUX) unset(FREEBSD) +unset(OPENBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) + +file(GLOB OPENBSD_PROBE "${CROSS_ROOTFS}/etc/signify/openbsd-*.pub") + if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) set(FREEBSD 1) +elseif(OPENBSD_PROBE) + set(CMAKE_SYSTEM_NAME OpenBSD) + set(OPENBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) @@ -52,7 +59,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu") endif() elseif(FREEBSD) - set(triple "aarch64-unknown-freebsd12") + set(TOOLCHAIN "aarch64-unknown-freebsd14") + elseif(OPENBSD) + set(TOOLCHAIN "aarch64-unknown-openbsd") endif() elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -67,6 +76,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6") else() set(TOOLCHAIN "arm-linux-gnueabihf") endif() +elseif(TARGET_ARCH_NAME STREQUAL "loongarch64") + set(CMAKE_SYSTEM_PROCESSOR "loongarch64") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/loongarch64-alpine-linux-musl) + set(TOOLCHAIN "loongarch64-alpine-linux-musl") + else() + set(TOOLCHAIN "loongarch64-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) @@ -101,7 +117,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu") endif() elseif(FREEBSD) - set(triple "x86_64-unknown-freebsd12") + set(TOOLCHAIN "x86_64-unknown-freebsd14") + elseif(OPENBSD) + set(TOOLCHAIN "x86_64-unknown-openbsd") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) @@ -118,7 +136,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu") endif() else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64 and x86 are supported!") endif() if(DEFINED ENV{TOOLCHAIN}) @@ -141,13 +159,29 @@ if(TIZEN) else() find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() - - message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}") - include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++) include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN}) endif() +function(locate_toolchain_exec exec var) + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + find_program(EXEC_LOCATION_${exec} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) +endfunction() + if(ANDROID) if(TARGET_ARCH_NAME STREQUAL "arm") set(ANDROID_ABI armeabi-v7a) @@ -167,77 +201,41 @@ if(ANDROID) # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(FREEBSD) +elseif(FREEBSD OR OPENBSD) # we cross-compile by instructing clang - set(CMAKE_C_COMPILER_TARGET ${triple}) - set(CMAKE_CXX_COMPILER_TARGET ${triple}) - set(CMAKE_ASM_COMPILER_TARGET ${triple}) + set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") elseif(ILLUMOS) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") include_directories(SYSTEM ${CROSS_ROOTFS}/include) - set(TOOLSET_PREFIX ${TOOLCHAIN}-) - function(locate_toolchain_exec exec var) - string(TOUPPER ${exec} EXEC_UPPERCASE) - if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") - set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) - return() - endif() - - find_program(EXEC_LOCATION_${exec} - NAMES - "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" - "${TOOLSET_PREFIX}${exec}") - - if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") - message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") - endif() - set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) - endfunction() - - set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - - set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") - - set(TOOLSET_PREFIX ${TOOLCHAIN}-) - function(locate_toolchain_exec exec var) - string(TOUPPER ${exec} EXEC_UPPERCASE) - if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") - set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) - return() - endif() - - find_program(EXEC_LOCATION_${exec} - NAMES - "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" - "${TOOLSET_PREFIX}${exec}") - - if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") - message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") - endif() - set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) - endfunction() - set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") - - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") + if ("$ENV{CCC_CC}" MATCHES ".*gcc.*") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + else() + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + endif() + # let CMake set up the correct search paths include(Platform/Haiku) else() @@ -307,7 +305,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD AND NOT OPENBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e33743105635..a5be41db6906 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 36dbd45e1ce8..e6ba4ee28c11 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -68,14 +68,14 @@ function InstallDarcCli { fi fi - local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" + local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/dotnet.cmd b/eng/common/dotnet.cmd new file mode 100644 index 000000000000..527fa4bb38fb --- /dev/null +++ b/eng/common/dotnet.cmd @@ -0,0 +1,7 @@ +@echo off + +:: This script is used to install the .NET SDK. +:: It will also invoke the SDK with any provided arguments. + +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet.ps1""" %*" +exit /b %ErrorLevel% diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1 new file mode 100644 index 000000000000..ce4ea40730a0 --- /dev/null +++ b/eng/common/dotnet.ps1 @@ -0,0 +1,12 @@ +# This script is used to install the .NET SDK. +# It will also invoke the SDK with any provided arguments. + +. $PSScriptRoot\tools.ps1 +$dotnetRoot = InitializeDotNetCli -install:$true + +# Invoke acquired SDK with args if they are provided +if ($args.count -gt 0) { + $env:DOTNET_NOLOGO=1 + & "$dotnetRoot\dotnet.exe" $args + ExitWithExitCode $LASTEXITCODE +} diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh new file mode 100755 index 000000000000..2ef68235675f --- /dev/null +++ b/eng/common/dotnet.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script is used to install the .NET SDK. +# It will also invoke the SDK with any provided arguments. + +source="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +source $scriptroot/tools.sh +InitializeDotNetCli true # install + +# Invoke acquired SDK with args if they are provided +if [[ $# > 0 ]]; then + __dotnetDir=${_InitializeDotNetCli} + dotnetPath=${__dotnetDir}/dotnet + ${dotnetPath} "$@" +fi diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 524aaa57f2b7..fa1cdc2b3007 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -33,15 +33,27 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$wxlFilesV3 = @() +$wxlFilesV5 = @() $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them if (-not $wxlFiles) { $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files if ($wxlEnFiles) { - $wxlFiles = @() - $wxlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $content = Get-Content $_.FullName -Raw + + # Split files on schema to select different parser settings in the generated project. + if ($content -like "*http://wixtoolset.org/schemas/v4/wxl*") + { + $wxlFilesV5 += Copy-Item $_.FullName -Destination $destinationFile -PassThru + } + elseif ($content -like "*http://schemas.microsoft.com/wix/2006/localization*") + { + $wxlFilesV3 += Copy-Item $_.FullName -Destination $destinationFile -PassThru + } + } } } @@ -114,7 +126,32 @@ $locJson = @{ CloneLanguageSet = "WiX_CloneLanguages" LssFiles = @( "wxl_loc.lss" ) LocItems = @( - $wxlFiles | ForEach-Object { + $wxlFilesV3 | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "P210WxlSchemaV4.lss" ) + LocItems = @( + $wxlFilesV5 | ForEach-Object { $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { diff --git a/eng/common/native/LocateNativeCompiler.targets b/eng/common/native/LocateNativeCompiler.targets new file mode 100644 index 000000000000..028b33d94441 --- /dev/null +++ b/eng/common/native/LocateNativeCompiler.targets @@ -0,0 +1,27 @@ + + + + + clang + $(ROOTFS_DIR) + + + + + + + + $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) + <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) + lld + + + diff --git a/eng/common/native/NativeAotSupported.props b/eng/common/native/NativeAotSupported.props new file mode 100644 index 000000000000..559a6663929e --- /dev/null +++ b/eng/common/native/NativeAotSupported.props @@ -0,0 +1,26 @@ + + + + + <_NativeAotSupportedOS Condition=" + '$(TargetOS)' != 'browser' and + '$(TargetOS)' != 'haiku' and + '$(TargetOS)' != 'illumos' and + '$(TargetOS)' != 'netbsd' and + '$(TargetOS)' != 'solaris' + ">true + + + <_NativeAotSupportedArch Condition=" + '$(TargetArchitecture)' != 'wasm' and + ('$(TargetArchitecture)' != 'x86' or '$(TargetOS)' == 'windows') + ">true + + true + + + diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 83ea7aab0e08..8fc6d2fec78d 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -39,6 +39,8 @@ getNonPortableDistroRid() # $rootfsDir can be empty. freebsd-version is a shell script and should always work. __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif [ "$targetOs" = "openbsd" ]; then + nonPortableRid="openbsd.$(uname -r)-${targetArch}" elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh new file mode 100755 index 000000000000..aff839fa0974 --- /dev/null +++ b/eng/common/native/install-dependencies.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -e + +# This is a simple script primarily used for CI to install necessary dependencies +# +# Usage: +# +# ./install-dependencies.sh + +os="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + +if [ -z "$os" ]; then + . "$(dirname "$0")"/init-os-and-arch.sh +fi + +case "$os" in + linux) + if [ -e /etc/os-release ]; then + . /etc/os-release + fi + + if [ "$ID" = "debian" ] || [ "$ID_LIKE" = "debian" ]; then + apt update + + apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ + libssl-dev libkrb5-dev pigz cpio ninja-build file + + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then + pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file + elif [ "$ID" = "amzn" ]; then + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file + elif [ "$ID" = "alpine" ]; then + apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja file + else + echo "Unsupported distro. distro: $ID" + exit 1 + fi + ;; + + osx|maccatalyst|ios|iossimulator|tvos|tvossimulator) + echo "Installed xcode version: $(xcode-select -p)" + + export HOMEBREW_NO_INSTALL_CLEANUP=1 + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + # Skip brew update for now, see https://github.com/actions/setup-python/issues/577 + # brew update --preinstall + brew bundle --no-upgrade --file=- < Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludeCIBinaryLog When running on CI, allow no binary log (short: -nobl)" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." } @@ -34,10 +39,11 @@ function Print-Usage() { function Build([string]$target) { $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } $log = Join-Path $LogDir "$task$logSuffix.binlog" + $binaryLogArg = if ($binaryLog) { "/bl:$log" } else { "" } $outputPath = Join-Path $ToolsetDir "$task\" MSBuild $taskProject ` - /bl:$log ` + $binaryLogArg ` /t:$target ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` @@ -64,7 +70,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh new file mode 100755 index 000000000000..3270f83fa9a7 --- /dev/null +++ b/eng/common/sdk-task.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +show_usage() { + echo "Common settings:" + echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + echo " --restore Restore dependencies" + echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" + echo " --help Print help and exit" + echo "" + + echo "Advanced settings:" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" + echo " --noWarnAsError Do not warn as error" + echo "" + echo "Command line arguments not listed above are passed thru to msbuild." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +Build() { + local target=$1 + local log_suffix="" + [[ "$target" != "Execute" ]] && log_suffix=".$target" + local log="$log_dir/$task$log_suffix.binlog" + local binaryLogArg="" + [[ $binary_log == true ]] && binaryLogArg="/bl:$log" + local output_path="$toolset_dir/$task/" + + MSBuild "$taskProject" \ + $binaryLogArg \ + /t:"$target" \ + /p:Configuration="$configuration" \ + /p:RepoRoot="$repo_root" \ + /p:BaseIntermediateOutputPath="$output_path" \ + /v:"$verbosity" \ + $properties +} + +binary_log=true +configuration="Debug" +verbosity="minimal" +exclude_ci_binary_log=false +restore=false +help=false +properties='' +warnAsError=true + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --task) + task=$2 + shift 2 + ;; + --restore) + restore=true + shift 1 + ;; + --verbosity) + verbosity=$2 + shift 2 + ;; + --excludecibinarylog|--nobl) + binary_log=false + exclude_ci_binary_log=true + shift 1 + ;; + --noWarnAsError) + warnAsError=false + shift 1 + ;; + --help) + help=true + shift 1 + ;; + *) + properties="$properties $1" + shift 1 + ;; + esac +done + +ci=true + +if $help; then + show_usage + exit 0 +fi + +. "$scriptroot/tools.sh" +InitializeToolset + +if [[ -z "$task" ]]; then + Write-PipelineTelemetryError -Category 'Task' -Name 'MissingTask' -Message "Missing required parameter '-task '" + ExitWithExitCode 1 +fi + +taskProject=$(GetSdkTaskProject "$task") +if [[ ! -e "$taskProject" ]]; then + Write-PipelineTelemetryError -Category 'Task' -Name 'UnknownTask' -Message "Unknown task: $task" + ExitWithExitCode 1 +fi + +if $restore; then + Build "Restore" +fi + +Build "Execute" + + +ExitWithExitCode 0 diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 4585cfd6bba1..e5f543ea68c2 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index 4bf4cf41bd7c..e2b07a865f10 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -82,7 +82,6 @@ eng\common\ publish-build-artifacts.yml (logic) publish-pipeline-artifacts.yml (logic) component-governance.yml (shim) - generate-sbom.yml (shim) publish-logs.yml (shim) retain-build.yml (shim) send-to-helix.yml (shim) @@ -107,7 +106,6 @@ eng\common\ setup-maestro-vars.yml (logic) steps\ component-governance.yml (logic) - generate-sbom.yml (logic) publish-build-artifacts.yml (redirect) publish-logs.yml (logic) publish-pipeline-artifacts.yml (redirect) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 81ea7a261f2d..d68e9fbc2656 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,24 +1,15 @@ parameters: -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml parameters: is1ESPipeline: true - componentGovernanceSteps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion }} - BuildDropPath: ${{ parameters.buildDropPath }} - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - publishArtifacts: false - # publish artifacts # for 1ES managed templates, use the templateContext.output to handle multiple outputs. templateContext: @@ -26,11 +17,19 @@ jobs: outputs: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish pipeline artifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - condition: always() + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + continueOnError: true + - output: pipelineArtifact + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked continueOnError: true - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - output: pipelineArtifact @@ -39,17 +38,18 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish Logs - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - output: pipelineArtifact @@ -57,14 +57,20 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # BuildConfiguration is a non-production artifact - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + # V4 publishing: automatically publish staged artifacts as a pipeline artifact. + # The artifact name matches the SDK's FutureArtifactName ($(System.PhaseName)_Artifacts), + # which is encoded in the asset manifest for downstream publishing to discover. + # Jobs can opt in by setting enablePublishing: true. + - ${{ if and(eq(parameters.publishingVersion, 4), eq(parameters.enablePublishing, 'true')) }}: - output: pipelineArtifact - displayName: Publish SBOM manifest + displayName: 'Publish V4 pipeline artifacts' + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: '$(System.PhaseName)_Artifacts' continueOnError: true - targetPath: $(Build.ArtifactStagingDirectory)/sbom - artifactName: $(ARTIFACT_NAME) + retryCountOnTaskFailure: 10 # for any files being locked # add any outputs provided via root yaml - ${{ if ne(parameters.templateContext.outputs, '') }}: diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml index 100a3fc98493..fcf6637b2ebc 100644 --- a/eng/common/templates-official/steps/publish-build-artifacts.yml +++ b/eng/common/templates-official/steps/publish-build-artifacts.yml @@ -24,6 +24,10 @@ parameters: - name: is1ESPipeline type: boolean default: true + +- name: retryCountOnTaskFailure + type: string + default: 10 steps: - ${{ if ne(parameters.is1ESPipeline, true) }}: @@ -38,4 +42,5 @@ steps: PathtoPublish: ${{ parameters.pathToPublish }} ${{ if parameters.artifactName }}: ArtifactName: ${{ parameters.artifactName }} - + ${{ if parameters.retryCountOnTaskFailure }}: + retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml index 172f9f0fdc97..9e5981365e56 100644 --- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -24,5 +24,7 @@ steps: artifactName: ${{ parameters.args.artifactName }} ${{ if parameters.args.properties }}: properties: ${{ parameters.args.properties }} - ${{ if parameters.args.sbomEnabled }}: + ${{ if ne(parameters.args.sbomEnabled, '') }}: sbomEnabled: ${{ parameters.args.sbomEnabled }} + ${{ if ne(parameters.args.isProduction, '') }}: + isProduction: ${{ parameters.args.isProduction }} diff --git a/eng/common/templates-official/steps/source-index-stage1-publish.yml b/eng/common/templates-official/steps/source-index-stage1-publish.yml new file mode 100644 index 000000000000..9b8b80942b5c --- /dev/null +++ b/eng/common/templates-official/steps/source-index-stage1-publish.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index 1f308b24efc4..2cc3ae305d5a 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 +# image: windows.vs2026.amd64 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 5bdd3dd85fd2..5e261f34db42 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,12 +1,12 @@ parameters: enablePublishBuildArtifacts: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# CG related params, unused now and can eventually be removed + disableComponentGovernance: unused +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml @@ -21,31 +21,34 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - componentGovernanceSteps: - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + # we don't run CG in public + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable artifactPublishSteps: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish pipeline artifacts - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true - condition: always() + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: @@ -56,19 +59,19 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container - artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -79,4 +82,4 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml index 6428a98dfef6..605e602e94d1 100644 --- a/eng/common/templates/steps/publish-build-artifacts.yml +++ b/eng/common/templates/steps/publish-build-artifacts.yml @@ -25,6 +25,10 @@ parameters: type: string default: 'Container' +- name: retryCountOnTaskFailure + type: string + default: 10 + steps: - ${{ if eq(parameters.is1ESPipeline, true) }}: - 'eng/common/templates cannot be referenced from a 1ES managed template': error @@ -37,4 +41,6 @@ steps: PublishLocation: ${{ parameters.publishLocation }} PathtoPublish: ${{ parameters.pathToPublish }} ${{ if parameters.artifactName }}: - ArtifactName: ${{ parameters.artifactName }} \ No newline at end of file + ArtifactName: ${{ parameters.artifactName }} + ${{ if parameters.retryCountOnTaskFailure }}: + retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }} diff --git a/eng/common/templates/steps/source-index-stage1-publish.yml b/eng/common/templates/steps/source-index-stage1-publish.yml new file mode 100644 index 000000000000..182cec33a7bb --- /dev/null +++ b/eng/common/templates/steps/source-index-stage1-publish.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml new file mode 100644 index 000000000000..cdc6a28ff1f6 --- /dev/null +++ b/eng/common/templates/steps/vmr-sync.yml @@ -0,0 +1,186 @@ +### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet). +### They initialize the darc CLI and pull the new updates. +### Changes are applied locally onto the already cloned VMR (located in $vmrPath). + +parameters: +- name: targetRef + displayName: Target revision in dotnet/ to synchronize + type: string + default: $(Build.SourceVersion) + +- name: vmrPath + displayName: Path where the dotnet/dotnet is checked out to + type: string + default: $(Agent.BuildDirectory)/vmr + +- name: additionalSyncs + displayName: Optional list of package names whose repo's source will also be synchronized in the local VMR, e.g. NuGet.Protocol + type: object + default: [] + +steps: +- checkout: vmr + displayName: Clone dotnet/dotnet + path: vmr + clean: true + +- checkout: self + displayName: Clone $(Build.Repository.Name) + path: repo + fetchDepth: 0 + +# This step is needed so that when we get a detached HEAD / shallow clone, +# we still pull the commit into the temporary repo clone to use it during the sync. +# Also unshallow the clone so that forwardflow command would work. +- script: | + git branch repo-head + git rev-parse HEAD + displayName: Label PR commit + workingDirectory: $(Agent.BuildDirectory)/repo + +- script: | + git config --global user.name "dotnet-maestro[bot]" + git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" + displayName: Set git author to dotnet-maestro[bot] + workingDirectory: ${{ parameters.vmrPath }} + +- script: | + ./eng/common/vmr-sync.sh \ + --vmr ${{ parameters.vmrPath }} \ + --tmp $(Agent.TempDirectory) \ + --azdev-pat '$(AzdoToken)' \ + --ci \ + --debug + + if [ "$?" -ne 0 ]; then + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + fi + displayName: Sync repo into VMR (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + +- script: | + git config --global diff.astextplain.textconv echo + git config --system core.longpaths true + displayName: Configure Windows git (longpaths, astextplain) + condition: eq(variables['Agent.OS'], 'Windows_NT') + +- powershell: | + ./eng/common/vmr-sync.ps1 ` + -vmr ${{ parameters.vmrPath }} ` + -tmp $(Agent.TempDirectory) ` + -azdevPat '$(AzdoToken)' ` + -ci ` + -debugOutput + + if ($LASTEXITCODE -ne 0) { + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + } + displayName: Sync repo into VMR (Windows) + condition: eq(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + +- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - task: CopyFiles@2 + displayName: Collect failed patches + condition: failed() + inputs: + SourceFolder: '$(Agent.TempDirectory)' + Contents: '*.patch' + TargetFolder: '$(Build.ArtifactStagingDirectory)/FailedPatches' + + - publish: '$(Build.ArtifactStagingDirectory)/FailedPatches' + artifact: $(System.JobDisplayName)_FailedPatches + displayName: Upload failed patches + condition: failed() + +- ${{ each assetName in parameters.additionalSyncs }}: + # The vmr-sync script ends up staging files in the local VMR so we have to commit those + - script: + git commit --allow-empty -am "Forward-flow $(Build.Repository.Name)" + displayName: Commit local VMR changes + workingDirectory: ${{ parameters.vmrPath }} + + - script: | + set -ex + + echo "Searching for details of asset ${{ assetName }}..." + + # Use darc to get dependencies information + dependencies=$(./.dotnet/dotnet darc get-dependencies --name '${{ assetName }}' --ci) + + # Extract repository URL and commit hash + repository=$(echo "$dependencies" | grep 'Repo:' | sed 's/Repo:[[:space:]]*//' | head -1) + + if [ -z "$repository" ]; then + echo "##vso[task.logissue type=error]Asset ${{ assetName }} not found in the dependency list" + exit 1 + fi + + commit=$(echo "$dependencies" | grep 'Commit:' | sed 's/Commit:[[:space:]]*//' | head -1) + + echo "Updating the VMR from $repository / $commit..." + cd .. + git clone $repository ${{ assetName }} + cd ${{ assetName }} + git checkout $commit + git branch "sync/$commit" + + ./eng/common/vmr-sync.sh \ + --vmr ${{ parameters.vmrPath }} \ + --tmp $(Agent.TempDirectory) \ + --azdev-pat '$(dn-bot-all-orgs-code-r)' \ + --ci \ + --debug + + if [ "$?" -ne 0 ]; then + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + fi + displayName: Sync ${{ assetName }} into (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + + - powershell: | + $ErrorActionPreference = 'Stop' + + Write-Host "Searching for details of asset ${{ assetName }}..." + + $dependencies = .\.dotnet\dotnet darc get-dependencies --name '${{ assetName }}' --ci + + $repository = $dependencies | Select-String -Pattern 'Repo:\s+([^\s]+)' | Select-Object -First 1 + $repository -match 'Repo:\s+([^\s]+)' | Out-Null + $repository = $matches[1] + + if ($repository -eq $null) { + Write-Error "Asset ${{ assetName }} not found in the dependency list" + exit 1 + } + + $commit = $dependencies | Select-String -Pattern 'Commit:\s+([^\s]+)' | Select-Object -First 1 + $commit -match 'Commit:\s+([^\s]+)' | Out-Null + $commit = $matches[1] + + Write-Host "Updating the VMR from $repository / $commit..." + cd .. + git clone $repository ${{ assetName }} + cd ${{ assetName }} + git checkout $commit + git branch "sync/$commit" + + .\eng\common\vmr-sync.ps1 ` + -vmr ${{ parameters.vmrPath }} ` + -tmp $(Agent.TempDirectory) ` + -azdevPat '$(dn-bot-all-orgs-code-r)' ` + -ci ` + -debugOutput + + if ($LASTEXITCODE -ne 0) { + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + } + displayName: Sync ${{ assetName }} into (Windows) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 18693ea120d5..587770f0add4 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2022.amd64 +# demands: ImageOverride -equals windows.vs2026.amd64 variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - template: /eng/common/templates-official/variables/pool-providers.yml diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml new file mode 100644 index 000000000000..d24de935248a --- /dev/null +++ b/eng/common/templates/vmr-build-pr.yml @@ -0,0 +1,43 @@ +# This pipeline is used for running the VMR verification of the PR changes in repo-level PRs. +# +# It will run a full set of verification jobs defined in: +# https://github.com/dotnet/dotnet/blob/10060d128e3f470e77265f8490f5e4f72dae738e/eng/pipelines/templates/stages/vmr-build.yml#L27-L38 +# +# For repos that do not need to run the full set, you would do the following: +# +# 1. Copy this YML file to a repo-specific location, i.e. outside of eng/common. +# +# 2. Add `verifications` parameter to VMR template reference +# +# Examples: +# - For source-build stage 1 verification, add the following: +# verifications: [ "source-build-stage1" ] +# +# - For Windows only verifications, add the following: +# verifications: [ "unified-build-windows-x64", "unified-build-windows-x86" ] + +trigger: none +pr: none + +variables: +- template: /eng/common/templates/variables/pool-providers.yml@self + +- name: skipComponentGovernanceDetection # we run CG on internal builds only + value: true + +- name: Codeql.Enabled # we run CodeQL on internal builds only + value: false + +resources: + repositories: + - repository: vmr + type: github + name: dotnet/dotnet + endpoint: public + ref: refs/heads/main # Set to whatever VMR branch the PR build should insert into + +stages: +- template: /eng/pipelines/templates/stages/vmr-build.yml@vmr + parameters: + isBuiltFromVmr: false + scope: lite diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a06513a59407..977a2d4b1039 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,10 +65,8 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# True if the build is a product build -[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } - -[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } +# True when the build is running within the VMR. +[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false } function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null @@ -259,7 +257,20 @@ function Retry($downloadBlock, $maxRetries = 5) { function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' + $shouldDownload = $false + if (!(Test-Path $installScript)) { + $shouldDownload = $true + } else { + # Check if the script is older than 30 days + $fileAge = (Get-Date) - (Get-Item $installScript).LastWriteTime + if ($fileAge.Days -gt 30) { + Write-Host "Existing install script is too old, re-downloading..." + $shouldDownload = $true + } + } + + if ($shouldDownload) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" @@ -383,8 +394,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.12.0 - $defaultXCopyMSBuildVersion = '17.12.0' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 + $defaultXCopyMSBuildVersion = '18.0.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -533,7 +544,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - $vswhereVersion = '2.5.2' + # keep this in sync with the VSWhereVersion in DefaultVersions.props + $vswhereVersion = '3.1.7' } $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" @@ -541,7 +553,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir - Write-Host 'Downloading vswhere' + Write-Host "Downloading vswhere $vswhereVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -UseBasicParsing -OutFile $vswhereExe }) @@ -611,14 +624,7 @@ function InitializeBuildTool() { } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - # Use override if it exists - commonly set by source-build - if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { - $initializeBuildToolFramework="net9.0" - } else { - $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework - } - - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -627,7 +633,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS } } else { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 @@ -660,7 +666,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } @@ -782,26 +787,13 @@ function MSBuild() { $toolsetBuildProject = InitializeToolset $basePath = Split-Path -parent $toolsetBuildProject - $possiblePaths = @( - # new scripts need to work with old packages, so we need to look for the old names/versions - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll')) - ) - $selectedPath = $null - foreach ($path in $possiblePaths) { - if (Test-Path $path -PathType Leaf) { - $selectedPath = $path - break - } - } + $selectedPath = Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll') + if (-not $selectedPath) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.' + Write-PipelineTelemetryError -Category 'Build' -Message "Unable to find arcade sdk logger assembly: $selectedPath" ExitWithExitCode 1 } + $args += "/logger:$selectedPath" } @@ -832,6 +824,11 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + if ($env:MSBUILD_MT_ENABLED -eq "1") { + $cmdArgs += ' -mt' + } + if ($warnAsError) { $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true' } @@ -864,8 +861,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) { + # Skip this when the build is a child of the VMR build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 01b09b65796c..1b296f646c23 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -5,6 +5,9 @@ # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +# Build mode +source_build=${source_build:-false} + # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md # This flag is meant as a temporary opt-opt for the feature while validate it across @@ -58,7 +61,8 @@ use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} # True to use global NuGet cache instead of restoring packages to repository-local directory. -if [[ "$ci" == true ]]; then +# Keep in sync with NuGetPackageroot in Arcade SDK's RepositoryLayout.props. +if [[ "$ci" == true || "$source_build" == true ]]; then use_global_nuget_cache=${use_global_nuget_cache:-false} else use_global_nuget_cache=${use_global_nuget_cache:-true} @@ -68,8 +72,8 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# True if the build is a product build -product_build=${product_build:-false} +# True when the build is running within the VMR. +from_vmr=${from_vmr:-false} # Resolve any symlinks in the given path. function ResolvePath { @@ -296,8 +300,29 @@ function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" + local timestamp_file="$root/.dotnet-install.timestamp" + local should_download=false if [[ ! -a "$install_script" ]]; then + should_download=true + elif [[ -f "$timestamp_file" ]]; then + # Check if the script is older than 30 days using timestamp file + local download_time=$(cat "$timestamp_file" 2>/dev/null || echo "0") + local current_time=$(date +%s) + local age_seconds=$((current_time - download_time)) + + # 30 days = 30 * 24 * 60 * 60 = 2592000 seconds + if [[ $age_seconds -gt 2592000 ]]; then + echo "Existing install script is too old, re-downloading..." + should_download=true + fi + else + # No timestamp file exists, assume script is old and re-download + echo "No timestamp found for existing install script, re-downloading..." + should_download=true + fi + + if [[ "$should_download" == true ]]; then mkdir -p "$root" echo "Downloading '$install_script_url'" @@ -324,6 +349,9 @@ function GetDotNetInstallScript { ExitWithExitCode $exit_code } fi + + # Create timestamp file to track download time in seconds from epoch + date +%s > "$timestamp_file" fi # return value _GetDotNetInstallScript="$install_script" @@ -339,22 +367,14 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - # use override if it exists - commonly set by source-build - if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then - _InitializeBuildToolFramework="net9.0" - else - _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" - fi } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi @@ -451,25 +471,13 @@ function MSBuild { fi local toolset_dir="${_InitializeToolset%/*}" - # new scripts need to work with old packages, so we need to look for the old names/versions - local selectedPath= - local possiblePaths=() - possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - for path in "${possiblePaths[@]}"; do - if [[ -f $path ]]; then - selectedPath=$path - break - fi - done + local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" + if [[ -z "$selectedPath" ]]; then - Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly." + Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly: $selectedPath" ExitWithExitCode 1 fi + args+=( "-logger:$selectedPath" ) fi @@ -506,8 +514,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then + # Skip this when the build is a child of the VMR build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error @@ -518,7 +526,13 @@ function MSBuild-Core { } } - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + local mt_switch="" + if [[ "${MSBUILD_MT_ENABLED:-}" == "1" ]]; then + mt_switch="-mt" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -530,6 +544,13 @@ function GetDarc { fi "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version + darc_tool="$darc_path/darc" +} + +# Returns a full path to an Arcade SDK task project file. +function GetSdkTaskProject { + taskName=$1 + echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj" } ResolvePath "${BASH_SOURCE[0]}" diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1 new file mode 100755 index 000000000000..b37992d91cf0 --- /dev/null +++ b/eng/common/vmr-sync.ps1 @@ -0,0 +1,164 @@ +<# +.SYNOPSIS + +This script is used for synchronizing the current repository into a local VMR. +It pulls the current repository's code into the specified VMR directory for local testing or +Source-Build validation. + +.DESCRIPTION + +The tooling used for synchronization will clone the VMR repository into a temporary folder if +it does not already exist. These clones can be reused in future synchronizations, so it is +recommended to dedicate a folder for this to speed up re-runs. + +.EXAMPLE + Synchronize current repository into a local VMR: + ./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp" + +.PARAMETER tmpDir +Required. Path to the temporary folder where repositories will be cloned + +.PARAMETER vmrBranch +Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch + +.PARAMETER azdevPat +Optional. Azure DevOps PAT to use for cloning private repositories. + +.PARAMETER vmrDir +Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder + +.PARAMETER debugOutput +Optional. Enables debug logging in the darc vmr command. + +.PARAMETER ci +Optional. Denotes that the script is running in a CI environment. +#> +param ( + [Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")] + [string][Alias('t', 'tmp')]$tmpDir, + [string][Alias('b', 'branch')]$vmrBranch, + [string]$remote, + [string]$azdevPat, + [string][Alias('v', 'vmr')]$vmrDir, + [switch]$ci, + [switch]$debugOutput +) + +function Fail { + Write-Host "> $($args[0])" -ForegroundColor 'Red' +} + +function Highlight { + Write-Host "> $($args[0])" -ForegroundColor 'Cyan' +} + +$verbosity = 'verbose' +if ($debugOutput) { + $verbosity = 'debug' +} +# Validation + +if (-not $tmpDir) { + Fail "Missing -tmpDir argument. Please specify the path to the temporary folder where the repositories will be cloned" + exit 1 +} + +# Sanitize the input + +if (-not $vmrDir) { + $vmrDir = Join-Path $tmpDir 'dotnet' +} + +if (-not (Test-Path -Path $tmpDir -PathType Container)) { + New-Item -ItemType Directory -Path $tmpDir | Out-Null +} + +# Prepare the VMR + +if (-not (Test-Path -Path $vmrDir -PathType Container)) { + Highlight "Cloning 'dotnet/dotnet' into $vmrDir.." + git clone https://github.com/dotnet/dotnet $vmrDir + + if ($vmrBranch) { + git -C $vmrDir switch -c $vmrBranch + } +} +else { + if ((git -C $vmrDir diff --quiet) -eq $false) { + Fail "There are changes in the working tree of $vmrDir. Please commit or stash your changes" + exit 1 + } + + if ($vmrBranch) { + Highlight "Preparing $vmrDir" + git -C $vmrDir checkout $vmrBranch + git -C $vmrDir pull + } +} + +Set-StrictMode -Version Latest + +# Prepare darc + +Highlight 'Installing .NET, preparing the tooling..' +. .\eng\common\tools.ps1 +$dotnetRoot = InitializeDotNetCli -install:$true +$env:DOTNET_ROOT = $dotnetRoot +$darc = Get-Darc + +Highlight "Starting the synchronization of VMR.." + +# Synchronize the VMR +$versionDetailsPath = Resolve-Path (Join-Path $PSScriptRoot '..\Version.Details.xml') | Select-Object -ExpandProperty Path +[xml]$versionDetails = Get-Content -Path $versionDetailsPath +$repoName = $versionDetails.SelectSingleNode('//Source').Mapping +if (-not $repoName) { + Fail "Failed to resolve repo mapping from $versionDetailsPath" + exit 1 +} + +$darcArgs = ( + "vmr", "forwardflow", + "--tmp", $tmpDir, + "--$verbosity", + $vmrDir +) + +if ($ci) { + $darcArgs += ("--ci") +} + +if ($azdevPat) { + $darcArgs += ("--azdev-pat", $azdevPat) +} + +& "$darc" $darcArgs + +if ($LASTEXITCODE -eq 0) { + Highlight "Synchronization succeeded" +} +else { + Highlight "Failed to flow code into the local VMR. Falling back to resetting the VMR to match repo contents..." + git -C $vmrDir reset --hard + + $resetArgs = ( + "vmr", "reset", + "${repoName}:HEAD", + "--vmr", $vmrDir, + "--tmp", $tmpDir, + "--additional-remotes", "${repoName}:${repoRoot}" + ) + + & "$darc" $resetArgs + + if ($LASTEXITCODE -eq 0) { + Highlight "Successfully reset the VMR using 'darc vmr reset'" + } + else { + Fail "Synchronization of repo to VMR failed!" + Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." + Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." + Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 + } +} diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh new file mode 100755 index 000000000000..198caec59bd4 --- /dev/null +++ b/eng/common/vmr-sync.sh @@ -0,0 +1,227 @@ +#!/bin/bash + +### This script is used for synchronizing the current repository into a local VMR. +### It pulls the current repository's code into the specified VMR directory for local testing or +### Source-Build validation. +### +### The tooling used for synchronization will clone the VMR repository into a temporary folder if +### it does not already exist. These clones can be reused in future synchronizations, so it is +### recommended to dedicate a folder for this to speed up re-runs. +### +### USAGE: +### Synchronize current repository into a local VMR: +### ./vmr-sync.sh --tmp "$HOME/repos/tmp" "$HOME/repos/dotnet" +### +### Options: +### -t, --tmp, --tmp-dir PATH +### Required. Path to the temporary folder where repositories will be cloned +### +### -b, --branch, --vmr-branch BRANCH_NAME +### Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch +### +### --debug +### Optional. Turns on the most verbose logging for the VMR tooling +### +### --remote name:URI +### Optional. Additional remote to use during the synchronization +### This can be used to synchronize to a commit from a fork of the repository +### Example: 'runtime:https://github.com/yourfork/runtime' +### +### --azdev-pat +### Optional. Azure DevOps PAT to use for cloning private repositories. +### +### -v, --vmr, --vmr-dir PATH +### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +function print_help () { + sed -n '/^### /,/^$/p' "$source" | cut -b 5- +} + +COLOR_RED=$(tput setaf 1 2>/dev/null || true) +COLOR_CYAN=$(tput setaf 6 2>/dev/null || true) +COLOR_CLEAR=$(tput sgr0 2>/dev/null || true) +COLOR_RESET=uniquesearchablestring +FAILURE_PREFIX='> ' + +function fail () { + echo "${COLOR_RED}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_RED}}${COLOR_CLEAR}" >&2 +} + +function highlight () { + echo "${COLOR_CYAN}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_CYAN}}${COLOR_CLEAR}" +} + +tmp_dir='' +vmr_dir='' +vmr_branch='' +additional_remotes='' +verbosity=verbose +azdev_pat='' +ci=false + +while [[ $# -gt 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -t|--tmp|--tmp-dir) + tmp_dir=$2 + shift + ;; + -v|--vmr|--vmr-dir) + vmr_dir=$2 + shift + ;; + -b|--branch|--vmr-branch) + vmr_branch=$2 + shift + ;; + --remote) + additional_remotes="$additional_remotes $2" + shift + ;; + --azdev-pat) + azdev_pat=$2 + shift + ;; + --ci) + ci=true + ;; + -d|--debug) + verbosity=debug + ;; + -h|--help) + print_help + exit 0 + ;; + *) + fail "Invalid argument: $1" + print_help + exit 1 + ;; + esac + + shift +done + +# Validation + +if [[ -z "$tmp_dir" ]]; then + fail "Missing --tmp-dir argument. Please specify the path to the temporary folder where the repositories will be cloned" + exit 1 +fi + +# Sanitize the input + +if [[ -z "$vmr_dir" ]]; then + vmr_dir="$tmp_dir/dotnet" +fi + +if [[ ! -d "$tmp_dir" ]]; then + mkdir -p "$tmp_dir" +fi + +if [[ "$verbosity" == "debug" ]]; then + set -x +fi + +# Prepare the VMR + +if [[ ! -d "$vmr_dir" ]]; then + highlight "Cloning 'dotnet/dotnet' into $vmr_dir.." + git clone https://github.com/dotnet/dotnet "$vmr_dir" + + if [[ -n "$vmr_branch" ]]; then + git -C "$vmr_dir" switch -c "$vmr_branch" + fi +else + if ! git -C "$vmr_dir" diff --quiet; then + fail "There are changes in the working tree of $vmr_dir. Please commit or stash your changes" + exit 1 + fi + + if [[ -n "$vmr_branch" ]]; then + highlight "Preparing $vmr_dir" + git -C "$vmr_dir" checkout "$vmr_branch" + git -C "$vmr_dir" pull + fi +fi + +set -e + +# Prepare darc + +highlight 'Installing .NET, preparing the tooling..' +source "./eng/common/tools.sh" +InitializeDotNetCli true +GetDarc +dotnetDir=$( cd ./.dotnet/; pwd -P ) +dotnet=$dotnetDir/dotnet + +highlight "Starting the synchronization of VMR.." +set +e + +if [[ -n "$additional_remotes" ]]; then + additional_remotes="--additional-remotes $additional_remotes" +fi + +if [[ -n "$azdev_pat" ]]; then + azdev_pat="--azdev-pat $azdev_pat" +fi + +ci_arg='' +if [[ "$ci" == "true" ]]; then + ci_arg="--ci" +fi + +# Synchronize the VMR + +version_details_path=$(cd "$scriptroot/.."; pwd -P)/Version.Details.xml +repo_name=$(grep -m 1 '