[rush] Add a RUSH_BUILD_CACHE_OVERRIDE_JSON env var to override the build cache configuration. (#5154)

* Refactor BuildCacheConfiguration.

* Include a RUSH_BUILD_CACHE_OVERRIDE_JSON env var.

* fixup! Include a RUSH_BUILD_CACHE_OVERRIDE_JSON env var.

* fixup! Include a RUSH_BUILD_CACHE_OVERRIDE_JSON env var.

* Add support for RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH environment variable

* fixup! Add support for RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH environment variable

Author: iclanton

common/changes/@microsoft/rush/build-cache-override-support_2025-03-08-23-35.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add support for `RUSH_BUILD_CACHE_OVERRIDE_JSON` environment variable that takes a JSON string with the same format as the `common/config/build-cache.json` file and a `RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH` environment variable that takes a file path that can be used to override the build cache configuration that is normally provided by that file.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-lib.api.md

@@ -237,6 +237,8 @@ export class EnvironmentConfiguration {
     static get allowWarningsInSuccessfulBuild(): boolean;
     static get buildCacheCredential(): string | undefined;
     static get buildCacheEnabled(): boolean | undefined;
+    static get buildCacheOverrideJson(): string | undefined;
+    static get buildCacheOverrideJsonFilePath(): string | undefined;
     static get buildCacheWriteAllowed(): boolean | undefined;
     static get cobuildContextId(): string | undefined;
     static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined;
@@ -274,6 +276,8 @@ export const EnvironmentVariableNames: {
     readonly RUSH_BUILD_CACHE_CREDENTIAL: "RUSH_BUILD_CACHE_CREDENTIAL";
     readonly RUSH_BUILD_CACHE_ENABLED: "RUSH_BUILD_CACHE_ENABLED";
     readonly RUSH_BUILD_CACHE_WRITE_ALLOWED: "RUSH_BUILD_CACHE_WRITE_ALLOWED";
+    readonly RUSH_BUILD_CACHE_OVERRIDE_JSON: "RUSH_BUILD_CACHE_OVERRIDE_JSON";
+    readonly RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: "RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH";
     readonly RUSH_COBUILD_CONTEXT_ID: "RUSH_COBUILD_CONTEXT_ID";
     readonly RUSH_COBUILD_RUNNER_ID: "RUSH_COBUILD_RUNNER_ID";
     readonly RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED";

libraries/rush-lib/src/api/BuildCacheConfiguration.ts

@@ -2,7 +2,6 @@
 // See LICENSE in the project root for license information.
 
 import { createHash } from 'node:crypto';
-import * as path from 'path';
 
 import {
   JsonFile,
@@ -18,7 +17,7 @@ import { FileSystemBuildCacheProvider } from '../logic/buildCache/FileSystemBuil
 import { RushConstants } from '../logic/RushConstants';
 import type { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider';
 import { RushUserConfiguration } from './RushUserConfiguration';
-import { EnvironmentConfiguration } from './EnvironmentConfiguration';
+import { EnvironmentConfiguration, EnvironmentVariableNames } from './EnvironmentConfiguration';
 import {
   CacheEntryId,
   type IGenerateCacheEntryIdOptions,
@@ -83,14 +82,14 @@ interface IBuildCacheConfigurationOptions {
   cloudCacheProvider: ICloudBuildCacheProvider | undefined;
 }
 
+const BUILD_CACHE_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);
+
 /**
  * Use this class to load and save the "common/config/rush/build-cache.json" config file.
  * This file provides configuration options for cached project build output.
  * @beta
  */
 export class BuildCacheConfiguration {
-  private static _jsonSchema: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);
-
   /**
    * Indicates whether the build cache feature is enabled.
    * Typically it is enabled in the build-cache.json config file.
@@ -146,11 +145,12 @@ export class BuildCacheConfiguration {
     rushConfiguration: RushConfiguration,
     rushSession: RushSession
   ): Promise<BuildCacheConfiguration | undefined> {
-    const jsonFilePath: string = BuildCacheConfiguration.getBuildCacheConfigFilePath(rushConfiguration);
-    if (!FileSystem.exists(jsonFilePath)) {
-      return undefined;
-    }
-    return await BuildCacheConfiguration._loadAsync(jsonFilePath, terminal, rushConfiguration, rushSession);
+    const { buildCacheConfiguration } = await BuildCacheConfiguration._tryLoadInternalAsync(
+      terminal,
+      rushConfiguration,
+      rushSession
+    );
+    return buildCacheConfiguration;
   }
 
   /**
@@ -162,51 +162,85 @@ export class BuildCacheConfiguration {
     rushConfiguration: RushConfiguration,
     rushSession: RushSession
   ): Promise<BuildCacheConfiguration> {
-    const jsonFilePath: string = BuildCacheConfiguration.getBuildCacheConfigFilePath(rushConfiguration);
-    if (!FileSystem.exists(jsonFilePath)) {
+    const { buildCacheConfiguration, jsonFilePath } = await BuildCacheConfiguration._tryLoadInternalAsync(
+      terminal,
+      rushConfiguration,
+      rushSession
+    );
+
+    if (!buildCacheConfiguration) {
       terminal.writeErrorLine(
         `The build cache feature is not enabled. This config file is missing:\n` + jsonFilePath
       );
       terminal.writeLine(`\nThe Rush website documentation has instructions for enabling the build cache.`);
       throw new AlreadyReportedError();
     }
 
-    const buildCacheConfiguration: BuildCacheConfiguration = await BuildCacheConfiguration._loadAsync(
-      jsonFilePath,
-      terminal,
-      rushConfiguration,
-      rushSession
-    );
-
     if (!buildCacheConfiguration.buildCacheEnabled) {
       terminal.writeErrorLine(
         `The build cache feature is not enabled. You can enable it by editing this config file:\n` +
           jsonFilePath
       );
       throw new AlreadyReportedError();
     }
+
     return buildCacheConfiguration;
   }
 
   /**
    * Gets the absolute path to the build-cache.json file in the specified rush workspace.
    */
   public static getBuildCacheConfigFilePath(rushConfiguration: RushConfiguration): string {
-    return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.buildCacheFilename);
+    return `${rushConfiguration.commonRushConfigFolder}/${RushConstants.buildCacheFilename}`;
+  }
+
+  private static async _tryLoadInternalAsync(
+    terminal: ITerminal,
+    rushConfiguration: RushConfiguration,
+    rushSession: RushSession
+  ): Promise<{ buildCacheConfiguration: BuildCacheConfiguration | undefined; jsonFilePath: string }> {
+    const jsonFilePath: string = BuildCacheConfiguration.getBuildCacheConfigFilePath(rushConfiguration);
+    const buildCacheConfiguration: BuildCacheConfiguration | undefined =
+      await BuildCacheConfiguration._tryLoadAsync(jsonFilePath, terminal, rushConfiguration, rushSession);
+    return { buildCacheConfiguration, jsonFilePath };
   }
 
-  private static async _loadAsync(
+  private static async _tryLoadAsync(
     jsonFilePath: string,
     terminal: ITerminal,
     rushConfiguration: RushConfiguration,
     rushSession: RushSession
-  ): Promise<BuildCacheConfiguration> {
-    const buildCacheJson: IBuildCacheJson = await JsonFile.loadAndValidateAsync(
-      jsonFilePath,
-      BuildCacheConfiguration._jsonSchema
-    );
-    const rushUserConfiguration: RushUserConfiguration = await RushUserConfiguration.initializeAsync();
+  ): Promise<BuildCacheConfiguration | undefined> {
+    let buildCacheJson: IBuildCacheJson;
+    const buildCacheOverrideJson: string | undefined = EnvironmentConfiguration.buildCacheOverrideJson;
+    if (buildCacheOverrideJson) {
+      buildCacheJson = JsonFile.parseString(buildCacheOverrideJson);
+      BUILD_CACHE_JSON_SCHEMA.validateObject(
+        buildCacheJson,
+        `${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON} environment variable`
+      );
+    } else {
+      const buildCacheOverrideJsonFilePath: string | undefined =
+        EnvironmentConfiguration.buildCacheOverrideJsonFilePath;
+      if (buildCacheOverrideJsonFilePath) {
+        buildCacheJson = await JsonFile.loadAndValidateAsync(
+          buildCacheOverrideJsonFilePath,
+          BUILD_CACHE_JSON_SCHEMA
+        );
+      } else {
+        try {
+          buildCacheJson = await JsonFile.loadAndValidateAsync(jsonFilePath, BUILD_CACHE_JSON_SCHEMA);
+        } catch (e) {
+          if (!FileSystem.isNotExistError(e)) {
+            throw e;
+          } else {
+            return undefined;
+          }
+        }
+      }
+    }
 
+    const rushUserConfiguration: RushUserConfiguration = await RushUserConfiguration.initializeAsync();
     let innerGetCacheEntryId: GetCacheEntryIdFunction;
     try {
       innerGetCacheEntryId = CacheEntryId.parsePattern(buildCacheJson.cacheEntryNamePattern);
@@ -218,7 +252,9 @@ export class BuildCacheConfiguration {
     }
 
     const { cacheHashSalt = '', cacheProvider } = buildCacheJson;
-    const salt: string = `${RushConstants.buildCacheVersion}${cacheHashSalt ? `${RushConstants.hashDelimiter}${cacheHashSalt}` : ''}`;
+    const salt: string = `${RushConstants.buildCacheVersion}${
+      cacheHashSalt ? `${RushConstants.hashDelimiter}${cacheHashSalt}` : ''
+    }`;
     // Extend the cache entry id with to salt the hash
     // This facilitates forcing cache invalidation either when the build cache version changes (new version of Rush)
     // or when the user-side salt changes (need to purge bad cache entries, plugins including additional files)

libraries/rush-lib/src/api/EnvironmentConfiguration.ts

@@ -144,6 +144,34 @@ export const EnvironmentVariableNames = {
    */
   RUSH_BUILD_CACHE_WRITE_ALLOWED: 'RUSH_BUILD_CACHE_WRITE_ALLOWED',
 
+  /**
+   * Set this environment variable to a JSON string to override the build cache configuration that normally lives
+   * at `common/config/rush/build-cache.json`.
+   *
+   * This is useful for testing purposes, or for OSS repos that are have a local-only cache, but can have
+   * a different cache configuration in CI/CD pipelines.
+   *
+   * @remarks
+   * This is similar to {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH}, but it allows you to specify
+   * a JSON string instead of a file path. The two environment variables are mutually exclusive, meaning you can
+   * only use one of them at a time.
+   */
+  RUSH_BUILD_CACHE_OVERRIDE_JSON: 'RUSH_BUILD_CACHE_OVERRIDE_JSON',
+
+  /**
+   * Set this environment variable to the path to a `build-cache.json` file to override the build cache configuration
+   * that normally lives at `common/config/rush/build-cache.json`.
+   *
+   * This is useful for testing purposes, or for OSS repos that are have a local-only cache, but can have
+   * a different cache configuration in CI/CD pipelines.
+   *
+   * @remarks
+   * This is similar to {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON}, but it allows you to specify
+   * a file path instead of a JSON string. The two environment variables are mutually exclusive, meaning you can
+   * only use one of them at a time.
+   */
+  RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: 'RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH',
+
   /**
    * Setting this environment variable opts into running with cobuilds. The context id should be the same across
    * multiple VMs, but changed when it is a new round of cobuilds.
@@ -250,6 +278,10 @@ export class EnvironmentConfiguration {
 
   private static _buildCacheWriteAllowed: boolean | undefined;
 
+  private static _buildCacheOverrideJson: string | undefined;
+
+  private static _buildCacheOverrideJsonFilePath: string | undefined;
+
   private static _cobuildContextId: string | undefined;
 
   private static _cobuildRunnerId: string | undefined;
@@ -360,6 +392,24 @@ export class EnvironmentConfiguration {
     return EnvironmentConfiguration._buildCacheWriteAllowed;
   }
 
+  /**
+   * If set, overrides the build cache configuration that normally lives at `common/config/rush/build-cache.json`.
+   * See {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON}
+   */
+  public static get buildCacheOverrideJson(): string | undefined {
+    EnvironmentConfiguration._ensureValidated();
+    return EnvironmentConfiguration._buildCacheOverrideJson;
+  }
+
+  /**
+   * If set, overrides the build cache configuration that normally lives at `common/config/rush/build-cache.json`.
+   * See {@link EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH}
+   */
+  public static get buildCacheOverrideJsonFilePath(): string | undefined {
+    EnvironmentConfiguration._ensureValidated();
+    return EnvironmentConfiguration._buildCacheOverrideJsonFilePath;
+  }
+
   /**
    * Provides a determined cobuild context id if configured
    * See {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID}
@@ -517,6 +567,16 @@ export class EnvironmentConfiguration {
             break;
           }
 
+          case EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON: {
+            EnvironmentConfiguration._buildCacheOverrideJson = value;
+            break;
+          }
+
+          case EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: {
+            EnvironmentConfiguration._buildCacheOverrideJsonFilePath = value;
+            break;
+          }
+
           case EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID: {
             EnvironmentConfiguration._cobuildContextId = value;
             break;
@@ -578,6 +638,17 @@ export class EnvironmentConfiguration {
       );
     }
 
+    if (
+      EnvironmentConfiguration._buildCacheOverrideJsonFilePath &&
+      EnvironmentConfiguration._buildCacheOverrideJson
+    ) {
+      throw new Error(
+        `Environment variable ${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH} and ` +
+          `${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON} are mutually exclusive. ` +
+          `Only one may be specified.`
+      );
+    }
+
     // See doc comment for EnvironmentConfiguration._getRushGlobalFolderOverride().
     EnvironmentConfiguration._rushGlobalFolderOverride =
       EnvironmentConfiguration._getRushGlobalFolderOverride(process.env);