Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Add a new `provideNpmrcCredentialsViaEnvironment` experiment. PNPM 10.34.2 and newer ignore `${VAR}` tokens that appear in credentials and registry URLs in a project `.npmrc` file, which broke the practice of supplying registry credentials via environment variables in CI. When this experiment is enabled, Rush expands those tokens itself, passing credentials to PNPM using `npm_config_*` environment variables instead of writing them to the generated `.npmrc` file.",
"type": "minor"
}
]
}
1 change: 1 addition & 0 deletions common/reviews/api/rush-lib.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,7 @@ export interface IExperimentsJson {
omitAppleDoubleFilesFromBuildCache?: boolean;
omitImportersFromPreventManualShrinkwrapChanges?: boolean;
printEventHooksOutputToConsole?: boolean;
provideNpmrcCredentialsViaEnvironment?: boolean;
rushAlerts?: boolean;
strictChangefileValidation?: boolean;
useDirectFileTransfersForBuildCache?: boolean;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,5 +141,16 @@
* must implement the optional file-based methods for this to take effect; otherwise it falls back to the
* buffer-based approach.
*/
/*[LINE "HYPOTHETICAL"]*/ "useDirectFileTransfersForBuildCache": true
/*[LINE "HYPOTHETICAL"]*/ "useDirectFileTransfersForBuildCache": true,

/**
* PNPM 10.34.2 and newer ignore "${VAR}" tokens that appear in credentials and registry URLs in a
* project or workspace .npmrc file, because such files are normally committed to Git. Rush generates
* "common/temp/.npmrc", which PNPM classifies as a project file even though it is not committed, so
* PNPM discards those settings and prints a warning. If true, when using PNPM, Rush expands those
* tokens itself: credentials are passed to PNPM using "npm_config_*" environment variables instead of
* being written to the generated .npmrc file, and non-secret settings such as registry URLs are
* written to the generated file with their values already expanded.
*/
/*[LINE "HYPOTHETICAL"]*/ "provideNpmrcCredentialsViaEnvironment": true
}
14 changes: 14 additions & 0 deletions libraries/rush-lib/src/api/ExperimentsConfiguration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,20 @@ export interface IExperimentsJson {
* effect; otherwise it falls back to the buffer-based approach.
*/
useDirectFileTransfersForBuildCache?: boolean;

/**
* If true, when using PNPM, Rush resolves the `${VAR}` tokens that appear in credentials and
* registry URLs in the `.npmrc` file, instead of relying on PNPM to expand them. Credentials are
* passed to PNPM using `npm_config_*` environment variables and are not written to the generated
* `.npmrc` file.
*
* @remarks
* PNPM 10.34.2 and newer ignore `${VAR}` tokens in credentials and registry URLs that come from a
* project or workspace `.npmrc` file, because such files are normally committed to Git. Rush
* generates `common/temp/.npmrc`, which PNPM classifies as a project file even though it is not
* committed, so without this experiment PNPM discards those settings and prints a warning.
*/
provideNpmrcCredentialsViaEnvironment?: boolean;
}

const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);
Expand Down
14 changes: 14 additions & 0 deletions libraries/rush-lib/src/cli/RushPnpmCommandLineParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import type { IBuiltInPluginConfiguration } from '../pluginFramework/PluginLoade
import type { BaseInstallManager } from '../logic/base/BaseInstallManager';
import type { IInstallManagerOptions } from '../logic/base/BaseInstallManagerTypes';
import { Utilities } from '../utilities/Utilities';
import { getNpmrcEnvironmentVariables } from '../utilities/npmrcUtilities';
import { InstallHelpers } from '../logic/installManager/InstallHelpers';
import type { Subspace } from '../api/Subspace';
import type { PnpmOptionsConfiguration } from '../logic/pnpm/PnpmOptionsConfiguration';
import { PnpmWorkspaceFile } from '../logic/pnpm/PnpmWorkspaceFile';
Expand Down Expand Up @@ -476,6 +478,18 @@ export class RushPnpmCommandLineParser {
}
}

// Provide any credentials that "rush install" moved out of the generated .npmrc file.
// See the "provideNpmrcCredentialsViaEnvironment" experiment.
if (InstallHelpers.shouldProvideNpmrcCredentialsViaEnvironment(rushConfiguration)) {
const npmrcEnvironmentVariables: Record<string, string> | undefined = getNpmrcEnvironmentVariables({
npmrcFolder: workspaceFolder,
supportEnvVarFallbackSyntax: rushConfiguration.isPnpm
});
for (const [envKey, envValue] of Object.entries(npmrcEnvironmentVariables ?? {})) {
pnpmEnvironmentMap.set(envKey, envValue);
}
}

let onStdoutStreamChunk: ((chunk: string) => string | void) | undefined;
switch (this._commandName) {
case 'patch': {
Expand Down
30 changes: 28 additions & 2 deletions libraries/rush-lib/src/logic/Autoinstaller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { Colorize } from '@rushstack/terminal';

import { AsyncRecycler } from '../utilities/AsyncRecycler';
import { Utilities } from '../utilities/Utilities';
import { getNpmrcEnvironmentVariables } from '../utilities/npmrcUtilities';
import type { RushConfiguration } from '../api/RushConfiguration';
import { PackageJsonEditor } from '../api/PackageJsonEditor';
import { InstallHelpers } from './installManager/InstallHelpers';
Expand Down Expand Up @@ -143,7 +144,10 @@ export class Autoinstaller {
Utilities.syncNpmrc({
sourceNpmrcFolder: this._rushConfiguration.commonRushConfigFolder,
targetNpmrcFolder: autoinstallerFullPath,
supportEnvVarFallbackSyntax: this._rushConfiguration.isPnpm
supportEnvVarFallbackSyntax: this._rushConfiguration.isPnpm,
moveSensitiveSettingsToEnvironment: InstallHelpers.shouldProvideNpmrcCredentialsViaEnvironment(
this._rushConfiguration
)
});

this._logIfConsoleOutputIsNotRestricted(
Expand All @@ -154,6 +158,7 @@ export class Autoinstaller {
command: this._rushConfiguration.packageManagerToolFilename,
args: ['install', '--frozen-lockfile'],
workingDirectory: autoinstallerFullPath,
environment: this._getPackageManagerEnvironment(autoinstallerFullPath),
keepEnvironment: true
});

Expand Down Expand Up @@ -229,13 +234,17 @@ export class Autoinstaller {
Utilities.syncNpmrc({
sourceNpmrcFolder: this._rushConfiguration.commonRushConfigFolder,
targetNpmrcFolder: this.folderFullPath,
supportEnvVarFallbackSyntax: this._rushConfiguration.isPnpm
supportEnvVarFallbackSyntax: this._rushConfiguration.isPnpm,
moveSensitiveSettingsToEnvironment: InstallHelpers.shouldProvideNpmrcCredentialsViaEnvironment(
this._rushConfiguration
)
});

await Utilities.executeCommandAsync({
command: this._rushConfiguration.packageManagerToolFilename,
args: ['install'],
workingDirectory: this.folderFullPath,
environment: this._getPackageManagerEnvironment(this.folderFullPath),
keepEnvironment: true
});

Expand Down Expand Up @@ -278,4 +287,21 @@ export class Autoinstaller {
console.log(message ?? '');
}
}

/**
* Returns the environment to invoke the package manager with, or `undefined` to inherit this
* process's environment. See the `provideNpmrcCredentialsViaEnvironment` experiment.
*/
private _getPackageManagerEnvironment(npmrcFolder: string): NodeJS.ProcessEnv | undefined {
if (!InstallHelpers.shouldProvideNpmrcCredentialsViaEnvironment(this._rushConfiguration)) {
return undefined;
}

const npmrcEnvironmentVariables: Record<string, string> | undefined = getNpmrcEnvironmentVariables({
npmrcFolder,
supportEnvVarFallbackSyntax: this._rushConfiguration.isPnpm
});

return npmrcEnvironmentVariables && { ...process.env, ...npmrcEnvironmentVariables };
}
}
5 changes: 4 additions & 1 deletion libraries/rush-lib/src/logic/base/BaseInstallManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,10 @@ export abstract class BaseInstallManager {
targetNpmrcFolder: subspace.getSubspaceTempFolderPath(),
linesToPrepend: extraNpmrcLines,
createIfMissing: this.rushConfiguration.subspacesFeatureEnabled,
supportEnvVarFallbackSyntax: this.rushConfiguration.isPnpm
supportEnvVarFallbackSyntax: this.rushConfiguration.isPnpm,
moveSensitiveSettingsToEnvironment: InstallHelpers.shouldProvideNpmrcCredentialsViaEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Credentials are removed from .npmrc for all PNPM installs, but legacy RushInstallManager never supplies npmrcFolder to getPackageManagerEnvironment(). Private-registry installs fail when useWorkspaces: false. Pass the subspace temp folder as done by WorkspaceInstallManager.

this.rushConfiguration
)
});
this._syncNpmrcAlreadyCalled = true;

Expand Down
44 changes: 43 additions & 1 deletion libraries/rush-lib/src/logic/installManager/InstallHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type { IConfigurationEnvironment } from '../base/BasePackageManagerOption
import type { PnpmOptionsConfiguration } from '../pnpm/PnpmOptionsConfiguration';
import { PnpmWorkspaceFile } from '../pnpm/PnpmWorkspaceFile';
import { merge } from '../../utilities/objectUtilities';
import { getNpmrcEnvironmentVariables } from '../../utilities/npmrcUtilities';
import type { Subspace } from '../../api/Subspace';
import { RushConstants } from '../RushConstants';

Expand Down Expand Up @@ -377,10 +378,32 @@ export class InstallHelpers {
};
}

/**
* Returns true if Rush (rather than PNPM) should expand the `${VAR}` tokens that appear in
* credentials and registry URLs in the `.npmrc` file. See the
* `provideNpmrcCredentialsViaEnvironment` experiment.
*/
public static shouldProvideNpmrcCredentialsViaEnvironment(rushConfiguration: RushConfiguration): boolean {
// Only PNPM refuses to expand these tokens, and only PNPM's "npm_config_*" environment variable
// name normalization has been validated here.
const {
isPnpm,
experimentsConfiguration: {
configuration: { provideNpmrcCredentialsViaEnvironment = false }
}
} = rushConfiguration;
return isPnpm && provideNpmrcCredentialsViaEnvironment;
}

/**
* Returns the environment that the package manager should be invoked with, including any
* credentials that were moved out of the generated `.npmrc` file in `npmrcFolder`.
*/
public static getPackageManagerEnvironment(
rushConfiguration: RushConfiguration,
options: {
debug?: boolean;
npmrcFolder?: string;
} = {}
): NodeJS.ProcessEnv {
let configurationEnvironment: IConfigurationEnvironment | undefined = undefined;
Expand All @@ -393,7 +416,26 @@ export class InstallHelpers {
configurationEnvironment = rushConfiguration.yarnOptions?.environmentVariables;
}

return _mergeEnvironmentVariables(process.env, configurationEnvironment, options);
const packageManagerEnvironment: NodeJS.ProcessEnv = _mergeEnvironmentVariables(
process.env,
configurationEnvironment,
options
);

const { npmrcFolder } = options;
const shouldProvideCredentials: boolean =
InstallHelpers.shouldProvideNpmrcCredentialsViaEnvironment(rushConfiguration);
if (npmrcFolder !== undefined && shouldProvideCredentials) {
Object.assign(
packageManagerEnvironment,
getNpmrcEnvironmentVariables({
npmrcFolder,
supportEnvVarFallbackSyntax: rushConfiguration.isPnpm
})
);
}

return packageManagerEnvironment;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ export class WorkspaceInstallManager extends BaseInstallManager {

const packageManagerEnv: NodeJS.ProcessEnv = InstallHelpers.getPackageManagerEnvironment(
this.rushConfiguration,
this.options
{ ...this.options, npmrcFolder: subspace.getSubspaceTempFolderPath() }
);
Comment on lines 495 to 498
if (ConsoleTerminalProvider.supportsColor) {
packageManagerEnv.FORCE_COLOR = '1';
Expand Down
5 changes: 5 additions & 0 deletions libraries/rush-lib/src/schemas/experiments.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@
"useDirectFileTransfersForBuildCache": {
"description": "If true, the build cache will use file-based APIs to transfer cache entries to and from cloud storage. This avoids loading the entire cache entry into memory, which can prevent out-of-memory errors for large build outputs and allow cache entries to exceed the limit of a single Buffer. The cloud cache provider plugin must implement the optional file-based methods for this to take effect; otherwise it falls back to the buffer-based approach.",
"type": "boolean"
},

"provideNpmrcCredentialsViaEnvironment": {
"description": "If true, when using PNPM, Rush resolves the \"${VAR}\" tokens that appear in credentials and registry URLs in the .npmrc file, instead of relying on PNPM to expand them. Credentials are passed to PNPM using \"npm_config_*\" environment variables and are not written to the generated .npmrc file. PNPM 10.34.2 and newer ignore such tokens in a project .npmrc file, which otherwise breaks the recommended practice of supplying registry credentials via environment variables in CI.",
"type": "boolean"
}
},
"additionalProperties": false
Expand Down
Loading