diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c384ac458..159f0a02e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -110,10 +110,56 @@ jobs: PREFIX_DEV_READ_ONLY_TOKEN: ${{ secrets.PREFIX_DEV_READ_ONLY_TOKEN }} S3_ACCESS_KEY_ID: ${{ secrets.S3_UPLOAD_ACCESS_KEY_ID }} S3_SECRET_ACCESS_KEY: ${{ secrets.S3_UPLOAD_ACCESS_KEY_SECRET }} + - name: Stage x64 release binary + if: matrix.os == 'windows-latest' + shell: pwsh + run: | + $binary = Get-ChildItem target-pixi -Recurse -Filter rattler-build.exe | + Where-Object { $_.DirectoryName -match "\\release$" } | + Select-Object -First 1 + if ($null -eq $binary) { + throw "Could not find the x64 release binary." + } + New-Item -ItemType Directory -Path staging | Out-Null + Copy-Item $binary.FullName staging/rattler-build.exe + - name: Upload x64 release binary + if: matrix.os == 'windows-latest' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4.4.0 + with: + name: rattler-build-windows-x64-e2e + path: staging/rattler-build.exe + if-no-files-found: error - name: Show sccache stats if: always() run: pixi run sccache --show-stats + windows-arm-e2e-test: + name: End-to-end (windows-11-arm) + needs: e2e-test + # e2e-test is a matrix job. Run this when another platform fails, provided + # the workflow itself was not cancelled, so an unrelated failure does not + # skip validation of the successfully uploaded Windows x64 binary. + if: ${{ always() && !cancelled() }} + runs-on: windows-11-arm + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: recursive + persist-credentials: false + - uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0 + with: + cache: true + cache-write: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + - name: Download x64 release binary + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: rattler-build-windows-x64-e2e + path: bin + - name: Run Windows architecture E2E test + run: pixi run --platform win-64 --frozen pytest test/end-to-end/test_windows_architecture_execution.py -v + env: + RATTLER_BUILD_PATH: ${{ github.workspace }}\bin\rattler-build.exe + rebuild-test: name: Rebuild Test if: contains(github.event.pull_request.labels.*.name, 'needs-rebuild-tests') diff --git a/Cargo.lock b/Cargo.lock index 3a6604425..f7e679d5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5125,6 +5125,7 @@ dependencies = [ "tokio-util", "tracing", "which", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/rattler_build_core/src/env_vars.rs b/crates/rattler_build_core/src/env_vars.rs index 490e59de5..aefcb9630 100644 --- a/crates/rattler_build_core/src/env_vars.rs +++ b/crates/rattler_build_core/src/env_vars.rs @@ -203,6 +203,7 @@ pub fn os_vars( prefix: &Path, target_platform: &Platform, host_platform: &Platform, + build_platform: &Platform, env_isolation: EnvironmentIsolation, work_dir: &Path, ) -> HashMap> { @@ -260,13 +261,12 @@ pub fn os_vars( env_isolation, )); } - let build_platform = Platform::current(); if build_platform.is_windows() { - vars.extend(windows::env::default_env_vars_build(&build_platform)); + vars.extend(windows::env::default_env_vars_build(build_platform)); } else if build_platform.is_osx() { - vars.extend(macos::env::default_env_vars_build(&build_platform)); + vars.extend(macos::env::default_env_vars_build(build_platform)); } else if build_platform.is_linux() { - vars.extend(linux::env::default_env_vars_build(&build_platform)); + vars.extend(linux::env::default_env_vars_build(build_platform)); } if build_platform.is_windows() { @@ -502,6 +502,7 @@ mod test { prefix, &Platform::NoArch, &Platform::Win64, + &Platform::Win64, EnvironmentIsolation::Strict, work_dir, ); @@ -516,6 +517,25 @@ mod test { ); } + #[test] + fn build_vars_follow_configured_build_platform() { + let vars = os_vars( + Path::new("/some/prefix"), + &Platform::WinArm64, + &Platform::WinArm64, + &Platform::WinArm64, + EnvironmentIsolation::Strict, + Path::new("/some/work"), + ); + + let expected = + std::env::var("BUILD").unwrap_or_else(|_| "arm64-pc-windows-19.0.0".to_string()); + assert_eq!( + vars.get("BUILD").and_then(|value| value.as_deref()), + Some(expected.as_str()) + ); + } + /// noarch on a non-Windows host does not emit Windows target vars. #[test] fn test_noarch_non_windows_host_has_no_windows_vars() { @@ -526,6 +546,7 @@ mod test { prefix, &Platform::NoArch, &Platform::Linux64, + &Platform::Linux64, EnvironmentIsolation::Strict, work_dir, ); diff --git a/crates/rattler_build_core/src/package_test/run_test.rs b/crates/rattler_build_core/src/package_test/run_test.rs index 13d93c09c..9b6bf6390 100644 --- a/crates/rattler_build_core/src/package_test/run_test.rs +++ b/crates/rattler_build_core/src/package_test/run_test.rs @@ -12,7 +12,9 @@ use rattler_build_recipe::stage1::{ TestType, tests::{CommandsTest, DownstreamTest, PerlTest, PythonTest, PythonVersion, RTest, RubyTest}, }; -use rattler_build_script::{EnvironmentIsolation, Script, ScriptContent}; +use rattler_build_script::{ + EnvironmentIsolation, ExecutionContext, RuntimeEnv, Script, ScriptContent, +}; use rattler_build_types::NormalizedKey; use rattler_conda_types::{ Channel, ChannelUrl, MatchSpec, PackageName, PackageNameMatcher, ParseStrictness, Platform, @@ -204,16 +206,16 @@ impl Tests { ))) })?; - let platform = Platform::current(); let mut env_vars = env_vars::os_vars( environment, - &platform, - &platform, + &target_platform, + &host_platform, + &build_platform, config.env_isolation, tmp_dir.path(), ); if config.env_isolation == EnvironmentIsolation::None { - env_vars.retain(|key, _| key != ShellEnum::default().path_var(&platform)); + env_vars.retain(|key, _| key != ShellEnum::default().path_var(&build_platform)); } env_vars.extend(env_vars::test_vars( target_platform, @@ -223,7 +225,7 @@ impl Tests { env_vars.extend(env_vars::python_vars_from_records( resolved_records, environment, - platform, + host_platform, )); env_vars.extend(pkg_vars.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); env_vars.insert( @@ -231,6 +233,8 @@ impl Tests { Some(environment.to_string_lossy().to_string()), ); + let context = shared_test_context(environment, host_platform); + match self { Tests::Commands(path) => { let script = Script { @@ -243,8 +247,7 @@ impl Tests { env_vars, tmp_dir.path(), cwd, - environment, - None, + context, None:: Result>, None, config.env_isolation, @@ -264,8 +267,7 @@ impl Tests { env_vars, tmp_dir.path(), cwd, - environment, - None, + context, None:: Result>, None, config.env_isolation, @@ -343,6 +345,23 @@ pub struct TestConfiguration { pub env_isolation: EnvironmentIsolation, } +fn configured_test_platforms(config: &TestConfiguration) -> (Platform, Platform, Platform) { + let target_platform = config.target_platform.unwrap_or(Platform::current()); + let build_platform = config.current_platform.platform; + let host_platform = config + .host_platform + .as_ref() + .map(|platform| platform.platform) + .unwrap_or(target_platform); + (target_platform, build_platform, host_platform) +} + +/// Tests without a separate build environment execute programs from the test +/// prefix, so their wrapper architecture must match the package host platform. +fn shared_test_context(prefix: &Path, host_platform: Platform) -> ExecutionContext { + ExecutionContext::shared(RuntimeEnv::current(), prefix, host_platform, host_platform) +} + fn env_vars_from_package(index_json: &IndexJson) -> HashMap { let mut res = HashMap::new(); @@ -796,24 +815,26 @@ async fn run_python_test_inner( ..Script::default() }; - let platform = Platform::current(); + let (target_platform, build_platform, host_platform) = configured_test_platforms(config); let test_dir = prefix.join("test"); fs::create_dir_all(&test_dir)?; let test_env_vars = env_vars::os_vars( &test_prefix, - &platform, - &platform, + &target_platform, + &host_platform, + &build_platform, config.env_isolation, &test_dir, ); + let context = shared_test_context(&test_prefix, host_platform); + script .run_script( test_env_vars.clone(), &test_dir, path, - &test_prefix, - None, + context.clone(), None:: Result>, None, config.env_isolation, @@ -836,8 +857,7 @@ async fn run_python_test_inner( test_env_vars, path, path, - &test_prefix, - None, + context, None:: Result>, None, config.env_isolation, @@ -908,22 +928,23 @@ async fn run_perl_test( let test_folder = prefix.join("test_files"); fs::create_dir_all(&test_folder)?; - let platform = Platform::current(); + let (target_platform, build_platform, host_platform) = configured_test_platforms(config); let test_env_vars = env_vars::os_vars( &test_prefix, - &platform, - &platform, + &target_platform, + &host_platform, + &build_platform, config.env_isolation, &test_folder, ); + let context = shared_test_context(&test_prefix, host_platform); script .run_script( test_env_vars, &test_folder, path, - &test_prefix, - None, + context, None:: Result>, None, config.env_isolation, @@ -1021,16 +1042,16 @@ async fn run_commands_test( ))) })?; - let platform = Platform::current(); let mut env_vars = env_vars::os_vars( &run_prefix, - &platform, - &platform, + &target_platform, + &host_platform, + &build_platform, config.env_isolation, &test_dir, ); if config.env_isolation == EnvironmentIsolation::None { - env_vars.retain(|key, _| key != ShellEnum::default().path_var(&platform)); + env_vars.retain(|key, _| key != ShellEnum::default().path_var(&build_platform)); } env_vars.extend(env_vars::test_vars( target_platform, @@ -1040,7 +1061,7 @@ async fn run_commands_test( env_vars.extend(env_vars::python_vars_from_records( &resolved_records, &run_prefix, - platform, + host_platform, )); env_vars.extend(pkg_vars.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); env_vars.insert( @@ -1048,6 +1069,18 @@ async fn run_commands_test( Some(run_prefix.to_string_lossy().to_string()), ); + let context = if let Some(build_prefix) = build_prefix { + ExecutionContext::separate( + RuntimeEnv::current(), + build_prefix, + build_platform, + &run_prefix, + host_platform, + ) + } else { + shared_test_context(&run_prefix, host_platform) + }; + tracing::info!("Testing commands:"); commands_test .script @@ -1055,8 +1088,7 @@ async fn run_commands_test( env_vars, &test_dir, path, - &run_prefix, - build_prefix.as_ref(), + context, None:: Result>, None, config.env_isolation, @@ -1221,22 +1253,23 @@ async fn run_r_test( let test_folder = prefix.join("test_files"); fs::create_dir_all(&test_folder)?; - let platform = Platform::current(); + let (target_platform, build_platform, host_platform) = configured_test_platforms(config); let test_env_vars = env_vars::os_vars( &test_prefix, - &platform, - &platform, + &target_platform, + &host_platform, + &build_platform, config.env_isolation, &test_folder, ); + let context = shared_test_context(&test_prefix, host_platform); script .run_script( test_env_vars, &test_folder, path, - &test_prefix, - None, + context, None:: Result>, None, config.env_isolation, @@ -1302,22 +1335,23 @@ async fn run_ruby_test( let test_folder = prefix.join("test_files"); fs::create_dir_all(&test_folder)?; - let platform = Platform::current(); + let (target_platform, build_platform, host_platform) = configured_test_platforms(config); let test_env_vars = env_vars::os_vars( &test_prefix, - &platform, - &platform, + &target_platform, + &host_platform, + &build_platform, config.env_isolation, &test_folder, ); + let context = shared_test_context(&test_prefix, host_platform); script .run_script( test_env_vars, &test_folder, path, - &test_prefix, - None, + context, None:: Result>, None, config.env_isolation, @@ -1332,6 +1366,15 @@ async fn run_ruby_test( mod tests { use super::*; + #[test] + fn shared_test_context_executes_the_host_prefix_platform() { + let context = shared_test_context(Path::new("test-prefix"), Platform::WinArm64); + assert_eq!(context.build().path(), Path::new("test-prefix")); + assert_eq!(context.host().path(), Path::new("test-prefix")); + assert_eq!(context.build().platform(), Platform::WinArm64); + assert_eq!(context.host().platform(), Platform::WinArm64); + } + /// Verifies that a trailing-underscore version (the openssl ordering /// convention, e.g. `3.7_`) is assembled into a `MatchSpec` that matches the /// exact version, without degrading into the broken string re-parse result. diff --git a/crates/rattler_build_core/src/script.rs b/crates/rattler_build_core/src/script.rs index 2d26ba177..6e75c15ef 100644 --- a/crates/rattler_build_core/src/script.rs +++ b/crates/rattler_build_core/src/script.rs @@ -9,8 +9,8 @@ use rattler_build_jinja::Jinja; // Re-export from rattler_build_script pub use rattler_build_script::{ - ExecutionArgs, InterpreterError, ResolvedScriptContents, RuntimeEnv, SandboxArguments, - SandboxConfiguration, Script, ScriptContent, platform_script_extensions, + ExecutionArgs, ExecutionContext, InterpreterError, ResolvedScriptContents, RuntimeEnv, + SandboxArguments, SandboxConfiguration, Script, ScriptContent, platform_script_extensions, }; use crate::{ @@ -32,24 +32,47 @@ impl Output { let target_platform = self.build_configuration.target_platform; let host_platform = self.host_platform().platform; let env_isolation = self.build_configuration.env_isolation; + let context = if self.recipe.build().merge_build_and_host_envs { + ExecutionContext::shared( + RuntimeEnv::current(), + &host_prefix, + self.build_configuration.build_platform.platform, + host_platform, + ) + } else { + ExecutionContext::separate( + RuntimeEnv::current(), + &self.build_configuration.directories.build_prefix, + self.build_configuration.build_platform.platform, + &host_prefix, + host_platform, + ) + }; + let mut env_vars = env_vars::vars(self, "BUILD"); env_vars.extend(env_vars::os_vars( &host_prefix, &target_platform, &host_platform, + &self.build_configuration.build_platform.platform, env_isolation, &self.build_configuration.directories.work_dir, )); env_vars.extend(env_vars::env_vars_from_variant(self.variant())); + if let Some(architecture) = context.windows_processor_architecture() { + env_vars.insert( + "PROCESSOR_ARCHITECTURE".to_string(), + Some(architecture.to_string()), + ); + } + if let Some(wow64_architecture) = context.windows_processor_architecture_w6432() { + env_vars.insert( + "PROCESSOR_ARCHITEW6432".to_string(), + Some(wow64_architecture.unwrap_or_default().to_string()), + ); + } let jinja_renderer = self.jinja_renderer(); - - let build_prefix = if self.recipe.build().merge_build_and_host_envs { - None - } else { - Some(&self.build_configuration.directories.build_prefix) - }; - let work_dir = &self.build_configuration.directories.work_dir; Ok(ExecutionArgs { interpreter: self.recipe.build().script.interpreter.clone(), @@ -63,9 +86,7 @@ impl Output { .filter_map(|(k, v)| v.map(|v| (k, v))) .collect(), secrets: IndexMap::new(), - build_prefix: build_prefix.map(|p| p.to_owned()), - run_prefix: host_prefix, - runtime: RuntimeEnv::current(), + context, work_dir: work_dir.clone(), sandbox_config: self.build_configuration.sandbox_config().cloned(), env_isolation, @@ -102,11 +123,7 @@ impl Output { } let exec_args = self.prepare_build_script().await?; - let build_prefix = if self.recipe.build().merge_build_and_host_envs { - None - } else { - Some(&self.build_configuration.directories.build_prefix) - }; + let context = exec_args.context.clone(); // Create Jinja context with environment variables let mut jinja = Jinja::new(self.build_configuration.selector_config()) @@ -134,8 +151,7 @@ impl Output { .collect(), &self.build_configuration.directories.work_dir, &self.build_configuration.directories.recipe_dir, - &self.build_configuration.directories.host_prefix, - build_prefix, + context, Some(jinja_renderer), self.build_configuration.sandbox_config(), self.build_configuration.env_isolation, diff --git a/crates/rattler_build_core/src/staging.rs b/crates/rattler_build_core/src/staging.rs index 5d3bf70ff..49779bdb3 100644 --- a/crates/rattler_build_core/src/staging.rs +++ b/crates/rattler_build_core/src/staging.rs @@ -14,6 +14,7 @@ use miette::{Context, IntoDiagnostic}; use minijinja::Value; use rattler_build_jinja::{Jinja, Variable}; use rattler_build_recipe::stage1::{InheritsFrom, StagingCache}; +use rattler_build_script::{ExecutionContext, RuntimeEnv}; use rattler_build_types::NormalizedKey; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -280,6 +281,7 @@ impl Output { self.prefix(), &target_platform, &host_platform, + &self.build_configuration.build_platform.platform, self.build_configuration.env_isolation, &self.build_configuration.directories.work_dir, )); @@ -324,10 +326,21 @@ impl Output { jinja.render_str(template).map_err(|e| e.to_string()) }; - let build_prefix = if staging.build.merge_build_and_host_envs { - None + let context = if staging.build.merge_build_and_host_envs { + ExecutionContext::shared( + RuntimeEnv::current(), + &self.build_configuration.directories.host_prefix, + self.build_configuration.build_platform.platform, + host_platform, + ) } else { - Some(&self.build_configuration.directories.build_prefix) + ExecutionContext::separate( + RuntimeEnv::current(), + &self.build_configuration.directories.build_prefix, + self.build_configuration.build_platform.platform, + &self.build_configuration.directories.host_prefix, + host_platform, + ) }; staging @@ -337,8 +350,7 @@ impl Output { env_vars, &self.build_configuration.directories.work_dir, &self.build_configuration.directories.recipe_dir, - &self.build_configuration.directories.host_prefix, - build_prefix, + context, Some(jinja_renderer), self.build_configuration.sandbox_config(), self.build_configuration.env_isolation, diff --git a/crates/rattler_build_script/Cargo.toml b/crates/rattler_build_script/Cargo.toml index b6114c681..f781b17ae 100644 --- a/crates/rattler_build_script/Cargo.toml +++ b/crates/rattler_build_script/Cargo.toml @@ -52,6 +52,12 @@ which = { workspace = true, optional = true } minijinja = { workspace = true, optional = true } strsim = { workspace = true, optional = true } +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = [ + "Win32_System_SystemInformation", + "Win32_System_Threading", +] } + [dev-dependencies] tokio = { workspace = true, features = ["rt", "macros", "fs"] } insta = { workspace = true, features = ["yaml"] } diff --git a/crates/rattler_build_script/src/activation.rs b/crates/rattler_build_script/src/activation.rs index f97358074..7fe924d40 100644 --- a/crates/rattler_build_script/src/activation.rs +++ b/crates/rattler_build_script/src/activation.rs @@ -11,29 +11,48 @@ use rattler_shell::{ shell::{self, Shell}, }; -use crate::execution::ExecutionArgs; +use crate::{PrefixLayout, execution::ExecutionArgs}; /// Returns the shell-specific activation script sourced/called by the native wrapper. pub(crate) fn activation_script( args: &ExecutionArgs, shell_type: T, ) -> Result { - let platform = args.runtime.platform(); - let mut shell_script = shell::ShellScript::new(shell_type.clone(), platform); + let mut shell_script = + shell::ShellScript::new(shell_type.clone(), args.context.build().platform()); for (k, v) in args.env_vars.iter() { shell_script.set_env_var(k, v)?; } + + // `start /machine` changes the child `cmd.exe` architecture but inherits + // variables that describe the outer process. Set the values in the + // activation script, where the build environment is assembled, rather than + // mutating the execution arguments. `PROCESSOR_IDENTIFIER` intentionally + // remains untouched because it describes the physical processor. + if let Some(architecture) = args.context.windows_processor_architecture() { + shell_script.set_env_var("PROCESSOR_ARCHITECTURE", architecture)?; + } + if let Some(wow64_architecture) = args.context.windows_processor_architecture_w6432() { + // An empty value produces `set "PROCESSOR_ARCHITEW6432="` for cmd.exe, + // clearing the marker for a 64-bit child or native x86 process. + shell_script.set_env_var("PROCESSOR_ARCHITEW6432", wow64_architecture.unwrap_or(""))?; + } + // Re-entrancy marker: this way the preamble sources this file // once and nested shells skip re-sourcing it. shell_script.set_env_var("CONDA_BUILD", "1")?; - let host_prefix_activator = - Activator::from_path(&args.run_prefix, shell_type.clone(), platform)?; + let host_prefix_activator = Activator::from_path( + args.context.host().path(), + shell_type.clone(), + args.context.host().platform(), + )?; // Do not pass the host CONDA_PREFIX to the activation. When // CONDA_PREFIX is set (e.g. running inside a pixi/conda env), the // activator generates deactivation scripts for that environment. let current_env = args - .runtime + .context + .runtime() .vars() .map(|(k, v)| (k.to_owned(), v.to_owned())) .collect::>(); @@ -46,9 +65,12 @@ pub(crate) fn activation_script( let host_activation = host_prefix_activator.activation(activation_vars)?; - if let Some(build_prefix) = &args.build_prefix { - let build_prefix_activator = - Activator::from_path(build_prefix, shell_type.clone(), platform)?; + if args.context.layout() == PrefixLayout::Separate { + let build_prefix_activator = Activator::from_path( + args.context.build().path(), + shell_type.clone(), + args.context.build().platform(), + )?; let activation_vars = ActivationVariables { conda_prefix: None, path: None, @@ -73,7 +95,46 @@ mod tests { use rattler_shell::shell; use crate::execution::{EnvironmentIsolation, ExecutionArgs, ResolvedScriptContents}; - use crate::runtime::RuntimeEnv; + use crate::{ExecutionContext, runtime::RuntimeEnv}; + + /// The activation script must set the Windows architecture variables after + /// caller-provided environment values, because `/machine` changes the child + /// process but not its inherited environment. + #[test] + fn architecture_transition_normalizes_processor_environment() { + let tmp = tempfile::tempdir().unwrap(); + let prefix = tmp.path().join("prefix"); + fs_err::create_dir_all(&prefix).unwrap(); + let mut env_vars = IndexMap::new(); + env_vars.insert("PROCESSOR_ARCHITECTURE".to_string(), "AMD64".to_string()); + env_vars.insert( + "PROCESSOR_IDENTIFIER".to_string(), + "ARMv8 (64-bit) Family".to_string(), + ); + let args = ExecutionArgs { + script: ResolvedScriptContents::Missing, + interpreter: None, + env_vars, + secrets: IndexMap::new(), + context: ExecutionContext::shared( + RuntimeEnv::for_test(Platform::Win64), + &prefix, + Platform::WinArm64, + Platform::WinArm64, + ), + work_dir: tmp.path().to_path_buf(), + sandbox_config: None, + env_isolation: EnvironmentIsolation::Strict, + }; + + let script = super::activation_script(&args, shell::CmdExe).unwrap(); + assert!(script.contains(r#"@SET "PROCESSOR_ARCHITECTURE=ARM64""#)); + assert!(script.contains(r#"@SET "PROCESSOR_ARCHITEW6432=""#)); + assert!( + script.contains(r#"@SET "PROCESSOR_IDENTIFIER=ARMv8 (64-bit) Family""#), + "the host processor identifier must be preserved: {script}" + ); + } /// When a build prefix is present, both the run prefix and build prefix are /// activated and the generated script references both paths. @@ -90,9 +151,13 @@ mod tests { interpreter: None, env_vars: IndexMap::new(), secrets: IndexMap::new(), - runtime: RuntimeEnv::for_test(Platform::current()), - build_prefix: Some(build_prefix.clone()), - run_prefix: run_prefix.clone(), + context: ExecutionContext::separate( + RuntimeEnv::for_test(Platform::current()), + build_prefix.clone(), + Platform::current(), + run_prefix.clone(), + Platform::current(), + ), work_dir: tmp.path().to_path_buf(), sandbox_config: None, env_isolation: EnvironmentIsolation::None, diff --git a/crates/rattler_build_script/src/execution.rs b/crates/rattler_build_script/src/execution.rs index 794db0f3d..1e85c609b 100644 --- a/crates/rattler_build_script/src/execution.rs +++ b/crates/rattler_build_script/src/execution.rs @@ -4,9 +4,9 @@ //! [`generate_build_script`], executes them with [`run_script`], and provides //! subprocess output handling via [`run_process_with_replacements`]. -use crate::runtime::RuntimeEnv; use crate::sandbox::SandboxConfiguration; use crate::script::{Script, ScriptContent}; +use crate::{execution_context::ExecutionContext, runtime::RuntimeEnv}; use fs_err as fs; use futures::TryStreamExt; use indexmap::IndexMap; @@ -82,14 +82,8 @@ pub struct ExecutionArgs { /// Secrets to set as env vars and replace in the output pub secrets: IndexMap, - /// The environment rattler-build is running in: process environment - /// variables (including `PATH`) and the platform scripts execute on. - pub runtime: RuntimeEnv, - - /// The build prefix that should contain the interpreter to use - pub build_prefix: Option, - /// The prefix to use for the script execution - pub run_prefix: PathBuf, + /// Process and platform-aware build and host prefixes for this execution. + pub context: ExecutionContext, /// The working directory (`cwd`) in which the script should execute pub work_dir: PathBuf, @@ -107,14 +101,12 @@ impl ExecutionArgs { /// will be replaced with the actual variable name. pub(crate) fn replacements(&self, template: &str) -> HashMap { let mut replacements = HashMap::new(); - if let Some(build_prefix) = &self.build_prefix { - replacements.insert( - build_prefix.display().to_string(), - template.replace("((var))", "BUILD_PREFIX"), - ); - }; replacements.insert( - self.run_prefix.display().to_string(), + self.context.build().path().display().to_string(), + template.replace("((var))", "BUILD_PREFIX"), + ); + replacements.insert( + self.context.host().path().display().to_string(), template.replace("((var))", "PREFIX"), ); @@ -194,8 +186,7 @@ impl Script { env_vars: HashMap>, work_dir: &Path, recipe_dir: &Path, - run_prefix: &Path, - build_prefix: Option<&PathBuf>, + context: ExecutionContext, jinja_renderer: Option, sandbox_config: Option<&SandboxConfiguration>, env_isolation: EnvironmentIsolation, @@ -215,7 +206,7 @@ impl Script { crate::platform_script_extensions(), )?; - let runtime = RuntimeEnv::current(); + let runtime = context.runtime(); let secrets = self .secrets() @@ -233,7 +224,7 @@ impl Script { .collect::>(); let work_dir = if let Some(cwd) = self.cwd.as_ref() { - run_prefix.join(cwd) + context.host().path().join(cwd) } else { work_dir.to_owned() }; @@ -245,9 +236,7 @@ impl Script { interpreter: self.interpreter.clone(), env_vars, secrets, - build_prefix: build_prefix.map(|p| p.to_owned()), - run_prefix: run_prefix.to_owned(), - runtime, + context, work_dir, sandbox_config: sandbox_config.cloned(), env_isolation, @@ -475,7 +464,7 @@ fn section_script_filename(extension: &str, index: SectionIndex) -> String { pub(crate) async fn generate_build_script( args: &ExecutionArgs, ) -> Result { - let runner = crate::native_runner::native_runner(args.runtime.platform()); + let runner = crate::native_runner::native_runner(args.context.runtime().process_platform()); let shell = runner.shell(); let script_extension = shell.extension(); @@ -605,11 +594,7 @@ async fn build_section_body( }; // Resolve from the activated environment (build/host prefix, then PATH). - let executable = interpreter.resolve_executable( - args.build_prefix.as_deref(), - &args.run_prefix, - &args.runtime, - )?; + let executable = interpreter.resolve_executable(&args.context)?; // Quote so a prefix or script path with spaces survives the native shell. let mut command = vec![executable.to_string_lossy().into_owned()]; @@ -628,13 +613,13 @@ async fn build_section_body( /// Runs a script with the given execution arguments. pub(crate) async fn run_script(exec_args: ExecutionArgs) -> Result<(), crate::InterpreterError> { - let runner = crate::native_runner::native_runner(exec_args.runtime.platform()); + let runner = + crate::native_runner::native_runner(exec_args.context.runtime().process_platform()); let build_script_path = generate_build_script(&exec_args).await?; - let build_script_path_str = build_script_path.to_string_lossy().to_string(); - let cmd_args = runner.command_to_run_script(&build_script_path_str); + let command_spec = runner.command_to_run_script(&build_script_path, &exec_args.context); let output = crate::execution::run_process_with_replacements( - &cmd_args, + &command_spec, &exec_args.work_dir, &exec_args.replacements(runner.replacements_template()), &exec_args.env_vars, @@ -645,17 +630,13 @@ pub(crate) async fn run_script(exec_args: ExecutionArgs) -> Result<(), crate::In } else { None }, - &exec_args.runtime, + exec_args.context.runtime(), ) .await?; if !output.status.success() { let status_code = output.status.code().unwrap_or(1); - let debug_info = runner.debug_info( - &exec_args.work_dir, - &exec_args.run_prefix, - exec_args.build_prefix.as_deref(), - ); + let debug_info = runner.debug_info(&exec_args.work_dir, &exec_args.context); tracing::error!("Script failed with status {}", status_code); tracing::error!("{}", debug_info); return Err(crate::InterpreterError::ExecutionFailed( @@ -792,7 +773,7 @@ fn configure_subprocess_env( /// This is used to replace the host prefix with $PREFIX and the build prefix with $BUILD_PREFIX #[allow(clippy::too_many_arguments)] pub(crate) async fn run_process_with_replacements( - args: &[&str], + command_spec: &crate::native_runner::CommandSpec, cwd: &Path, replacements: &HashMap, env_vars: &IndexMap, @@ -820,8 +801,8 @@ pub(crate) async fn run_process_with_replacements( cmd.args(&sandbox_args); // Add the actual command to execute (as positional arguments) - cmd.arg(args[0]); - cmd.args(&args[1..]); + cmd.arg(&command_spec.program); + cmd.args(&command_spec.args); cmd } else { @@ -833,7 +814,7 @@ pub(crate) async fn run_process_with_replacements( )); } } else { - tokio::process::Command::new(args[0]) + tokio::process::Command::new(&command_spec.program) }; configure_subprocess_env(&mut command, env_vars, secrets, env_isolation, runtime); @@ -843,7 +824,7 @@ pub(crate) async fn run_process_with_replacements( // when using `pixi global install bash` the current work dir // causes some strange issues that are fixed when setting the `PWD` .env("PWD", cwd) - .args(&args[1..]) + .args(&command_spec.args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -925,6 +906,7 @@ pub(crate) async fn run_process_with_replacements( #[cfg(test)] mod tests { use super::*; + use crate::ExecutionContext; use rattler_conda_types::Platform; use tokio_util::bytes::BytesMut; @@ -943,9 +925,12 @@ mod tests { interpreter: None, env_vars: IndexMap::new(), secrets: IndexMap::new(), - runtime: RuntimeEnv::for_test(Platform::current()), - build_prefix: None, - run_prefix: prefix, + context: ExecutionContext::shared( + RuntimeEnv::for_test(Platform::current()), + prefix, + Platform::current(), + Platform::current(), + ), work_dir: tmp.path().to_path_buf(), sandbox_config: None, env_isolation: EnvironmentIsolation::None, @@ -1050,9 +1035,12 @@ mod tests { interpreter: None, env_vars: IndexMap::new(), secrets: IndexMap::new(), - runtime: RuntimeEnv::for_test(Platform::Win64), - build_prefix: None, - run_prefix: prefix, + context: ExecutionContext::shared( + RuntimeEnv::for_test(Platform::Win64), + prefix, + Platform::Win64, + Platform::Win64, + ), work_dir: tmp.path().to_path_buf(), sandbox_config: None, env_isolation: EnvironmentIsolation::None, @@ -1283,9 +1271,12 @@ mod tests { interpreter: interpreter.map(str::to_string), env_vars: IndexMap::new(), secrets: IndexMap::new(), - runtime: RuntimeEnv::current(), - build_prefix: None, - run_prefix, + context: ExecutionContext::shared( + RuntimeEnv::current(), + run_prefix, + Platform::current(), + Platform::current(), + ), work_dir, sandbox_config: None, env_isolation: EnvironmentIsolation::None, @@ -1508,7 +1499,9 @@ mod tests { None, ); let args = ExecutionArgs { - runtime: RuntimeEnv::for_test(Platform::Linux64), + context: args + .context + .with_runtime(RuntimeEnv::for_test(Platform::Linux64)), ..args }; generate_build_script(&args).await.unwrap(); @@ -1534,7 +1527,9 @@ mod tests { None, ); let args = ExecutionArgs { - runtime: RuntimeEnv::for_test(Platform::Linux64), + context: args + .context + .with_runtime(RuntimeEnv::for_test(Platform::Linux64)), ..args }; generate_build_script(&args).await.unwrap(); @@ -1560,7 +1555,9 @@ mod tests { None, ); let args = ExecutionArgs { - runtime: RuntimeEnv::for_test(Platform::Win64), + context: args + .context + .with_runtime(RuntimeEnv::for_test(Platform::Win64)), ..args }; generate_build_script(&args).await.unwrap(); diff --git a/crates/rattler_build_script/src/execution_context.rs b/crates/rattler_build_script/src/execution_context.rs new file mode 100644 index 000000000..12003f1d5 --- /dev/null +++ b/crates/rattler_build_script/src/execution_context.rs @@ -0,0 +1,186 @@ +//! Platform-aware prefix and process context for script execution. + +use std::path::{Path, PathBuf}; + +use rattler_conda_types::Platform; + +use crate::RuntimeEnv; + +/// A conda prefix together with the platform of the environment it contains. +#[derive(Debug, Clone)] +pub struct PrefixWithPlatform { + path: PathBuf, + platform: Platform, +} + +impl PrefixWithPlatform { + /// Creates a prefix execution descriptor. + pub fn new(path: impl Into, platform: Platform) -> Self { + Self { + path: path.into(), + platform, + } + } + + /// The prefix path. + pub fn path(&self) -> &Path { + &self.path + } + + /// The platform of the environment installed in this prefix. + pub fn platform(&self) -> Platform { + self.platform + } +} + +/// Whether build and host environments have distinct or shared prefixes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrefixLayout { + /// Build and host environments are separate prefixes. + Separate, + /// Build and host environments share one prefix and must be activated once. + Shared, +} + +/// Process and prefix information needed to execute a build or test script. +#[derive(Debug, Clone)] +pub struct ExecutionContext { + runtime: RuntimeEnv, + build: PrefixWithPlatform, + host: PrefixWithPlatform, + layout: PrefixLayout, +} + +impl ExecutionContext { + /// Creates a context with separate build and host prefixes. + pub fn separate( + runtime: RuntimeEnv, + build_path: impl Into, + build_platform: Platform, + host_path: impl Into, + host_platform: Platform, + ) -> Self { + Self { + runtime, + build: PrefixWithPlatform::new(build_path, build_platform), + host: PrefixWithPlatform::new(host_path, host_platform), + layout: PrefixLayout::Separate, + } + } + + /// Creates a context whose build and host environments share one prefix. + pub fn shared( + runtime: RuntimeEnv, + path: impl Into, + build_platform: Platform, + host_platform: Platform, + ) -> Self { + let path = path.into(); + Self { + runtime, + build: PrefixWithPlatform::new(path.clone(), build_platform), + host: PrefixWithPlatform::new(path, host_platform), + layout: PrefixLayout::Shared, + } + } + + /// The environment and architecture of the rattler-build process. + pub fn runtime(&self) -> &RuntimeEnv { + &self.runtime + } + + /// Returns a copy with a different rattler-build process runtime. + #[must_use] + pub fn with_runtime(mut self, runtime: RuntimeEnv) -> Self { + self.runtime = runtime; + self + } + + /// The prefix that supplies build tools and the platform they execute on. + pub fn build(&self) -> &PrefixWithPlatform { + &self.build + } + + /// The prefix that supplies host dependencies and the platform it represents. + pub fn host(&self) -> &PrefixWithPlatform { + &self.host + } + + /// Whether build and host prefixes are separate or shared. + pub fn layout(&self) -> PrefixLayout { + self.layout + } + + /// The processor architecture to expose in a Windows child process. + /// + /// Returns a value only when the Windows runner must switch to x86 or + /// between x64 and ARM64 with `start /machine`; otherwise the child + /// inherits its normal process environment. + pub fn windows_processor_architecture(&self) -> Option<&'static str> { + crate::native_runner::windows_machine_transition( + self.runtime.process_platform(), + self.build.platform(), + ) + .map(crate::native_runner::WindowsMachine::processor_architecture) + } + + /// The value to set for `PROCESSOR_ARCHITEW6432` in a switched child. + /// + /// `Some(None)` clears the marker for a 64-bit child. `Some(value)` sets + /// the detected native architecture for an x86 WOW64 child. `None` keeps + /// the inherited value when no transition is needed or native detection is + /// unavailable. + pub fn windows_processor_architecture_w6432(&self) -> Option> { + let machine = crate::native_runner::windows_machine_transition( + self.runtime.process_platform(), + self.build.platform(), + )?; + + match machine { + crate::native_runner::WindowsMachine::X86 => { + crate::native_runner::native_windows_machine() + .map(crate::native_runner::WindowsMachine::wow64_processor_architecture) + } + crate::native_runner::WindowsMachine::Amd64 + | crate::native_runner::WindowsMachine::Arm64 => Some(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn separate_context_retains_both_prefixes() { + let context = ExecutionContext::separate( + RuntimeEnv::for_test(Platform::Win64), + "build", + Platform::Win64, + "host", + Platform::WinArm64, + ); + + assert_eq!(context.layout(), PrefixLayout::Separate); + assert_eq!(context.build().path(), Path::new("build")); + assert_eq!(context.build().platform(), Platform::Win64); + assert_eq!(context.host().path(), Path::new("host")); + assert_eq!(context.host().platform(), Platform::WinArm64); + } + + #[test] + fn shared_context_uses_one_path_with_both_platforms() { + let context = ExecutionContext::shared( + RuntimeEnv::for_test(Platform::Win64), + "prefix", + Platform::Win64, + Platform::WinArm64, + ); + + assert_eq!(context.layout(), PrefixLayout::Shared); + assert_eq!(context.build().path(), Path::new("prefix")); + assert_eq!(context.host().path(), Path::new("prefix")); + assert_eq!(context.build().platform(), Platform::Win64); + assert_eq!(context.host().platform(), Platform::WinArm64); + } +} diff --git a/crates/rattler_build_script/src/interpreter/cmd_exe.rs b/crates/rattler_build_script/src/interpreter/cmd_exe.rs index 1a7d66639..c17b0f1a2 100644 --- a/crates/rattler_build_script/src/interpreter/cmd_exe.rs +++ b/crates/rattler_build_script/src/interpreter/cmd_exe.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use rattler_conda_types::Platform; use super::{InterpreterInvocation, InterpreterSearchScope}; -use crate::runtime::RuntimeEnv; +use crate::ExecutionContext; pub struct CmdExeInvocation; @@ -35,21 +35,19 @@ impl InterpreterInvocation for CmdExeInvocation { fn resolve_executable( &self, - build_prefix: Option<&Path>, - run_prefix: &Path, - runtime: &RuntimeEnv, + context: &ExecutionContext, ) -> Result { - let platform = runtime.platform(); + let platform = context.build().platform(); let scope = self.search_scope(&platform); if platform.is_windows() && scope.allows_system_fallback() - && let Some(comspec) = runtime.var("COMSPEC") + && let Some(comspec) = context.runtime().var("COMSPEC") && comspec.to_lowercase().contains("cmd.exe") { return Ok(PathBuf::from(comspec)); } - super::find_interpreter("cmd", build_prefix, run_prefix, runtime, scope) + super::find_interpreter("cmd", context, scope) .ok_or_else(|| super::InterpreterError::InterpreterNotFound("cmd".to_string())) } diff --git a/crates/rattler_build_script/src/interpreter/mod.rs b/crates/rattler_build_script/src/interpreter/mod.rs index c6547f33d..d5a87113f 100644 --- a/crates/rattler_build_script/src/interpreter/mod.rs +++ b/crates/rattler_build_script/src/interpreter/mod.rs @@ -21,7 +21,7 @@ use std::path::{Path, PathBuf}; use rattler_conda_types::Platform; use rattler_shell::activation::prefix_path_entries; -use crate::runtime::RuntimeEnv; +use crate::ExecutionContext; /// Describes interpreter execution and lookup errors. #[derive(Debug, thiserror::Error)] @@ -126,23 +126,24 @@ impl InterpreterSearchScope { pub(crate) fn find_interpreter( name: &str, - build_prefix: Option<&Path>, - run_prefix: &Path, - runtime: &RuntimeEnv, + context: &ExecutionContext, scope: InterpreterSearchScope, ) -> Option { + let runtime = context.runtime(); let exe_name = format!("{}{}", name, runtime.exe_suffix()); - let platform = runtime.platform(); let mut search_path = Vec::new(); if scope.search_build { - // When build and host are merged there is no separate build prefix; the - // run prefix is the build environment. - let build_env = build_prefix.unwrap_or(run_prefix); - search_path.extend(prefix_path_entries(build_env, &platform)); + search_path.extend(prefix_path_entries( + context.build().path(), + &context.build().platform(), + )); } if scope.search_host { - search_path.extend(prefix_path_entries(run_prefix, &platform)); + search_path.extend(prefix_path_entries( + context.host().path(), + &context.host().platform(), + )); } if scope.system_fallback { search_path.extend(std::env::split_paths(runtime.path())); @@ -151,11 +152,8 @@ pub(crate) fn find_interpreter( if search_path.is_empty() { return None; } - // The interpreter is resolved on the host filesystem, so use the host's path - // conventions throughout: `std::env::{split_paths,join_paths}` and `which` - // (path-list separator, `PATHEXT`, executable bit) all key off the host - // platform. `runtime.platform()` always equals the host here, so there is no - // cross-platform case to handle. + // Prefix paths use their configured platform conventions. System fallback + // uses the rattler-build process environment and host filesystem rules. which::which_in_global(exe_name, std::env::join_paths(search_path).ok()) .ok()? .next() @@ -212,18 +210,13 @@ pub(crate) trait InterpreterInvocation: Send + Sync { /// /// The default implementation provides shared lookup behavior; interpreters /// can override it for platform-specific behavior. - fn resolve_executable( - &self, - build_prefix: Option<&Path>, - run_prefix: &Path, - runtime: &RuntimeEnv, - ) -> Result { - let platform = runtime.platform(); + fn resolve_executable(&self, context: &ExecutionContext) -> Result { + let platform = context.build().platform(); let scope = self.search_scope(&platform); let mut unusable_candidate = None; for executable_name in self.executable_names(&platform) { - match find_interpreter(executable_name, build_prefix, run_prefix, runtime, scope) { + match find_interpreter(executable_name, context, scope) { Some(path) => match self.is_usable_executable(&path) { Ok(()) => return Ok(path), Err(err) => unusable_candidate = Some((path, err)), @@ -300,12 +293,10 @@ impl SelectedInterpreter { /// Resolve the executable, remapping internal errors to the user-facing name. pub(crate) fn resolve_executable( &self, - build_prefix: Option<&Path>, - run_prefix: &Path, - runtime: &RuntimeEnv, + context: &ExecutionContext, ) -> Result { self.invocation - .resolve_executable(build_prefix, run_prefix, runtime) + .resolve_executable(context) .map_err(|err| match err { InterpreterError::InterpreterNotFound(_) => { InterpreterError::InterpreterNotFound(self.user_name.clone()) @@ -323,7 +314,10 @@ impl SelectedInterpreter { #[cfg(test)] mod tests { use super::*; - use crate::execution::{ExecutionArgs, ResolvedScriptContents}; + use crate::{ + ExecutionContext, RuntimeEnv, + execution::{ExecutionArgs, ResolvedScriptContents}, + }; use fs_err as fs; use indexmap::IndexMap; use rattler_conda_types::Platform; @@ -341,15 +335,23 @@ mod tests { interpreter: interpreter.map(str::to_string), env_vars: IndexMap::new(), secrets: IndexMap::new(), - runtime: RuntimeEnv::current(), - build_prefix: None, - run_prefix, + context: ExecutionContext::shared( + RuntimeEnv::current(), + run_prefix, + Platform::current(), + Platform::current(), + ), work_dir, sandbox_config: None, env_isolation: crate::execution::EnvironmentIsolation::None, } } + fn shared_context(runtime: RuntimeEnv, prefix: &Path) -> ExecutionContext { + let platform = runtime.process_platform(); + ExecutionContext::shared(runtime, prefix, platform, platform) + } + fn native_build_script_path(work_dir: &Path) -> PathBuf { work_dir.join(if cfg!(windows) { "conda_build.bat" @@ -590,11 +592,7 @@ mod tests { let stub = RejectFirstStub; let resolved = stub - .resolve_executable( - Some(prefix.as_path()), - prefix.as_path(), - &RuntimeEnv::current(), - ) + .resolve_executable(&shared_context(RuntimeEnv::current(), &prefix)) .expect("second candidate should resolve"); assert_eq!(resolved, second); } @@ -613,11 +611,7 @@ mod tests { let stub = RejectFirstStub; let err = stub - .resolve_executable( - Some(prefix.as_path()), - prefix.as_path(), - &RuntimeEnv::current(), - ) + .resolve_executable(&shared_context(RuntimeEnv::current(), &prefix)) .expect_err("only candidate is rejected"); match err { InterpreterError::InvalidInterpreter { interpreter, .. } => { @@ -647,11 +641,7 @@ mod tests { }; let err = selected - .resolve_executable( - Some(prefix.as_path()), - prefix.as_path(), - &RuntimeEnv::current(), - ) + .resolve_executable(&shared_context(RuntimeEnv::current(), &prefix)) .expect_err("only candidate is rejected"); match err { InterpreterError::InvalidInterpreter { @@ -691,18 +681,15 @@ mod tests { let runtime = RuntimeEnv::for_test(Platform::current()) .with_var("PATH", path_dir.to_string_lossy().into_owned()); + let context = shared_context(runtime, &prefix); let found_via_path = find_interpreter( "rb_path_only_tool", - Some(prefix.as_path()), - prefix.as_path(), - &runtime, + &context, InterpreterSearchScope::build_and_host_with_system_fallback(), ); let found_build_only = find_interpreter( "rb_path_only_tool", - Some(prefix.as_path()), - prefix.as_path(), - &runtime, + &context, InterpreterSearchScope::build_only(), ); @@ -765,20 +752,23 @@ mod tests { // Empty runtime PATH so resolution can only come from a prefix. let runtime = RuntimeEnv::for_test(Platform::current()).with_var("PATH", ""); + let context = ExecutionContext::separate( + runtime, + &build_prefix, + Platform::current(), + &host_prefix, + Platform::current(), + ); let found = find_interpreter( "rb_host_tool", - Some(build_prefix.as_path()), - host_prefix.as_path(), - &runtime, + &context, InterpreterSearchScope::build_and_host_with_system_fallback(), ); assert_eq!(found.as_deref(), Some(tool.as_path())); let build_only = find_interpreter( "rb_host_tool", - Some(build_prefix.as_path()), - host_prefix.as_path(), - &runtime, + &context, InterpreterSearchScope::build_only(), ); assert!( @@ -811,11 +801,7 @@ mod tests { let second = create_fake_executable(&prefix, "stub_second"); let resolved = RejectFirstStub - .resolve_executable( - Some(prefix.as_path()), - prefix.as_path(), - &RuntimeEnv::current(), - ) + .resolve_executable(&shared_context(RuntimeEnv::current(), &prefix)) .expect("second candidate should resolve when the first is absent"); assert_eq!(resolved, second); } @@ -834,8 +820,8 @@ mod tests { let runtime = RuntimeEnv::for_test(Platform::Win64) .with_var("COMSPEC", fake_cmd.to_string_lossy().into_owned()); - let resolved = - super::cmd_exe::CmdExeInvocation.resolve_executable(None, tmp.path(), &runtime); + let resolved = super::cmd_exe::CmdExeInvocation + .resolve_executable(&shared_context(runtime, tmp.path())); assert_eq!(resolved.unwrap(), fake_cmd); } diff --git a/crates/rattler_build_script/src/lib.rs b/crates/rattler_build_script/src/lib.rs index 49d117d29..aa53ff070 100644 --- a/crates/rattler_build_script/src/lib.rs +++ b/crates/rattler_build_script/src/lib.rs @@ -22,6 +22,8 @@ mod activation; #[cfg(feature = "execution")] mod execution; #[cfg(feature = "execution")] +mod execution_context; +#[cfg(feature = "execution")] mod interpreter; #[cfg(feature = "execution")] mod native_runner; @@ -33,6 +35,8 @@ pub use execution::{ EnvironmentIsolation, ExecutionArgs, ResolvedScriptContents, create_build_script, }; #[cfg(feature = "execution")] +pub use execution_context::{ExecutionContext, PrefixLayout, PrefixWithPlatform}; +#[cfg(feature = "execution")] pub use interpreter::{InterpreterError, closest_interpreter}; #[cfg(feature = "execution")] pub use runtime::RuntimeEnv; diff --git a/crates/rattler_build_script/src/native_runner/bash.rs b/crates/rattler_build_script/src/native_runner/bash.rs index efe3fd3e9..a192da139 100644 --- a/crates/rattler_build_script/src/native_runner/bash.rs +++ b/crates/rattler_build_script/src/native_runner/bash.rs @@ -4,7 +4,8 @@ use std::path::Path; use indexmap::IndexMap; use rattler_shell::shell::{self, Shell}; -use super::NativeShellRunner; +use super::{CommandSpec, NativeShellRunner}; +use crate::{ExecutionContext, PrefixLayout}; pub(crate) struct BashNativeRunner; @@ -34,8 +35,12 @@ set -x ) } - fn command_to_run_script<'a>(&self, build_script_path: &'a str) -> Vec<&'a str> { - vec!["bash", build_script_path] + fn command_to_run_script( + &self, + build_script_path: &Path, + _context: &ExecutionContext, + ) -> CommandSpec { + CommandSpec::new("bash", [build_script_path.to_string_lossy().into_owned()]) } fn replacements_template(&self) -> &'static str { @@ -71,20 +76,18 @@ set -x } /// Returns reproduction instructions for the failed bash wrapper script. - fn debug_info( - &self, - work_dir: &Path, - run_prefix: &Path, - build_prefix: Option<&Path>, - ) -> String { + fn debug_info(&self, work_dir: &Path, context: &ExecutionContext) -> String { let mut output = String::new(); output.push_str("\nScript execution failed.\n\n"); output.push_str(&format!(" Work directory: {}\n", work_dir.display())); - output.push_str(&format!(" Prefix: {}\n", run_prefix.display())); + output.push_str(&format!(" Prefix: {}\n", context.host().path().display())); - if let Some(build_prefix) = build_prefix { - output.push_str(&format!(" Build prefix: {}\n", build_prefix.display())); + if context.layout() == PrefixLayout::Separate { + output.push_str(&format!( + " Build prefix: {}\n", + context.build().path().display() + )); } else { output.push_str(" Build prefix: None\n"); } diff --git a/crates/rattler_build_script/src/native_runner/cmd_exe.rs b/crates/rattler_build_script/src/native_runner/cmd_exe.rs index a27191254..fbb41f7fd 100644 --- a/crates/rattler_build_script/src/native_runner/cmd_exe.rs +++ b/crates/rattler_build_script/src/native_runner/cmd_exe.rs @@ -4,7 +4,8 @@ use std::path::Path; use indexmap::IndexMap; use rattler_shell::shell::{self, Shell}; -use super::NativeShellRunner; +use super::{CommandSpec, NativeShellRunner, windows_machine_transition}; +use crate::{ExecutionContext, PrefixLayout}; pub(crate) struct CmdExeNativeRunner; @@ -33,8 +34,65 @@ IF "%CONDA_BUILD%" == "" ( ) } - fn command_to_run_script<'a>(&self, build_script_path: &'a str) -> Vec<&'a str> { - vec!["cmd.exe", "/d", "/c", build_script_path] + fn command_to_run_script( + &self, + build_script_path: &Path, + context: &ExecutionContext, + ) -> CommandSpec { + if let Some(machine) = windows_machine_transition( + context.runtime().process_platform(), + context.build().platform(), + ) { + // `start /machine` selects the architecture of the child `cmd.exe`. + // It normally returns immediately, so `/wait` is required to obtain + // the script's status. `cmd /c` otherwise returns the status of the + // `start` command itself, hence the explicit delayed `ERRORLEVEL` + // expansion and `exit /b` after the child finishes. + // + // The outer process runs in `work_dir`, so only the generated file + // name is needed. Quote it when necessary so a changed or reused + // filename containing whitespace remains a single argument. + let script_name = build_script_path + .file_name() + .expect("generated build script has a filename") + .to_string_lossy(); + let script_name = crate::native_runner::quote_arg(&self.shell(), &script_name); + // `/machine x86` does not redirect an explicit `cmd.exe` lookup + // from System32. Launch the x86 command interpreter from SysWOW64 + // directly. SystemRoot is conventionally an unspaced system path, + // so keep it unquoted to avoid `start` treating it as a title. The + // other architectures use `cmd.exe`, whose image selection is + // handled by `/machine`. + let child_cmd = match machine { + crate::native_runner::WindowsMachine::X86 => r"%SystemRoot%\SysWOW64\cmd.exe", + crate::native_runner::WindowsMachine::Amd64 + | crate::native_runner::WindowsMachine::Arm64 => "cmd.exe", + }; + let command = format!( + "start /b /wait /machine {} {} /d /c {} & exit /b !ERRORLEVEL!", + machine.start_argument(), + child_cmd, + script_name, + ); + CommandSpec::new( + "cmd.exe", + [ + "/d".to_string(), + "/v:on".to_string(), + "/c".to_string(), + command, + ], + ) + } else { + CommandSpec::new( + "cmd.exe", + [ + "/d".to_string(), + "/c".to_string(), + build_script_path.to_string_lossy().into_owned(), + ], + ) + } } fn replacements_template(&self) -> &'static str { @@ -75,26 +133,30 @@ IF "%CONDA_BUILD%" == "" ( } /// Returns reproduction instructions for the failed cmd wrapper script. - fn debug_info( - &self, - work_dir: &Path, - run_prefix: &Path, - build_prefix: Option<&Path>, - ) -> String { + fn debug_info(&self, work_dir: &Path, context: &ExecutionContext) -> String { let mut output = String::new(); output.push_str("\nScript execution failed.\n\n"); output.push_str(&format!(" Work directory: {}\n", work_dir.display())); - output.push_str(&format!(" Prefix: {}\n", run_prefix.display())); + output.push_str(&format!(" Prefix: {}\n", context.host().path().display())); - if let Some(build_prefix) = build_prefix { - output.push_str(&format!(" Build prefix: {}\n", build_prefix.display())); + if context.layout() == PrefixLayout::Separate { + output.push_str(&format!( + " Build prefix: {}\n", + context.build().path().display() + )); } else { output.push_str(" Build prefix: None\n"); } + let command = self.command_to_run_script(&work_dir.join("conda_build.bat"), context); output.push_str("\nTo run the script manually, use the following command:\n"); - output.push_str(&format!(" cd {:?} && ./conda_build.bat\n\n", work_dir)); + output.push_str(&format!( + " cd {:?} && {} {}\n\n", + work_dir, + command.program, + command.args.join(" ") + )); output.push_str("To run commands interactively in the build environment:\n"); output.push_str(&format!(" cd {:?} && call build_env.bat", work_dir)); diff --git a/crates/rattler_build_script/src/native_runner/mod.rs b/crates/rattler_build_script/src/native_runner/mod.rs index c9d490cd7..7697f870b 100644 --- a/crates/rattler_build_script/src/native_runner/mod.rs +++ b/crates/rattler_build_script/src/native_runner/mod.rs @@ -10,10 +10,128 @@ mod cmd_exe; use std::path::Path; -use indexmap::IndexMap; use rattler_conda_types::Platform; + +use indexmap::IndexMap; use rattler_shell::shell::{Shell, ShellEnum}; +use crate::ExecutionContext; + +/// A process invocation with owned arguments. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CommandSpec { + pub(crate) program: String, + pub(crate) args: Vec, +} + +impl CommandSpec { + pub(crate) fn new( + program: impl Into, + args: impl IntoIterator>, + ) -> Self { + Self { + program: program.into(), + args: args.into_iter().map(Into::into).collect(), + } + } +} + +/// Requested Windows child process architecture for a supported transition. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WindowsMachine { + X86, + Amd64, + Arm64, +} + +impl WindowsMachine { + pub(crate) fn start_argument(self) -> &'static str { + match self { + Self::X86 => "x86", + Self::Amd64 => "amd64", + Self::Arm64 => "arm64", + } + } + + pub(crate) fn processor_architecture(self) -> &'static str { + match self { + Self::X86 => "x86", + Self::Amd64 => "AMD64", + Self::Arm64 => "ARM64", + } + } + + /// The `PROCESSOR_ARCHITEW6432` marker Windows exposes to an x86 child. + pub(crate) fn wow64_processor_architecture(self) -> Option<&'static str> { + (self != Self::X86).then(|| self.processor_architecture()) + } + + #[cfg(windows)] + fn from_image_file_machine(machine: u16) -> Option { + use windows_sys::Win32::System::SystemInformation::{ + IMAGE_FILE_MACHINE_AMD64, IMAGE_FILE_MACHINE_ARM64, IMAGE_FILE_MACHINE_I386, + }; + + match machine { + IMAGE_FILE_MACHINE_I386 => Some(Self::X86), + IMAGE_FILE_MACHINE_AMD64 => Some(Self::Amd64), + IMAGE_FILE_MACHINE_ARM64 => Some(Self::Arm64), + _ => None, + } + } +} + +/// Returns the requested child architecture when a supported Windows build +/// process transition is needed. Rattler-build ships x64 and ARM64 binaries, +/// and both can launch x86 build tools; x86 rattler-build processes are not +/// supported as cross-architecture launchers. +pub(crate) fn windows_machine_transition( + process_platform: Platform, + build_platform: Platform, +) -> Option { + match (process_platform, build_platform) { + (Platform::Win64, Platform::Win32) | (Platform::WinArm64, Platform::Win32) => { + Some(WindowsMachine::X86) + } + (Platform::Win64, Platform::WinArm64) => Some(WindowsMachine::Arm64), + (Platform::WinArm64, Platform::Win64) => Some(WindowsMachine::Amd64), + _ => None, + } +} + +/// Detects the native Windows machine architecture without affecting launch +/// selection. This is only used to reproduce the `PROCESSOR_ARCHITEW6432` +/// value that Windows exposes to x86 WOW64 processes. +#[cfg(windows)] +pub(crate) fn native_windows_machine() -> Option { + use windows_sys::Win32::System::{ + SystemInformation::IMAGE_FILE_MACHINE_UNKNOWN, + Threading::{GetCurrentProcess, IsWow64Process2}, + }; + + let mut process_machine = IMAGE_FILE_MACHINE_UNKNOWN; + let mut native_machine = IMAGE_FILE_MACHINE_UNKNOWN; + // `IsWow64Process2` is available on every Windows version that supports + // `start /machine`. A failure leaves the inherited WOW64 marker untouched. + if unsafe { + IsWow64Process2( + GetCurrentProcess(), + &mut process_machine, + &mut native_machine, + ) + } == 0 + { + return None; + } + + WindowsMachine::from_image_file_machine(native_machine) +} + +#[cfg(not(windows))] +pub(crate) fn native_windows_machine() -> Option { + None +} + /// Defines platform-native wrapper execution. pub(crate) trait NativeShellRunner: Send + Sync { /// Returns the shell syntax used for the generated native wrapper script. @@ -26,8 +144,12 @@ pub(crate) trait NativeShellRunner: Send + Sync { /// Returns the shell preamble inserted at the top of `conda_build.*`. fn preamble(&self, activation_script_path: &Path) -> String; - /// Returns process argv used to execute the generated native wrapper script. - fn command_to_run_script<'a>(&self, build_script_path: &'a str) -> Vec<&'a str>; + /// Returns the process invocation used to execute the generated native wrapper script. + fn command_to_run_script( + &self, + build_script_path: &Path, + context: &ExecutionContext, + ) -> CommandSpec; /// Returns the replacement template used when streaming process output. fn replacements_template(&self) -> &'static str; @@ -49,8 +171,7 @@ pub(crate) trait NativeShellRunner: Send + Sync { ) -> Result; /// Returns human-readable reproduction instructions shown when execution fails. - fn debug_info(&self, work_dir: &Path, run_prefix: &Path, build_prefix: Option<&Path>) - -> String; + fn debug_info(&self, work_dir: &Path, context: &ExecutionContext) -> String; } /// Selects the native wrapper shell for the given platform: `cmd.exe` on @@ -92,7 +213,8 @@ pub(crate) fn quote_arg(shell: &ShellEnum, arg: &str) -> String { #[cfg(test)] mod tests { - use super::{native_runner, quote_arg}; + use super::{WindowsMachine, native_runner, quote_arg, windows_machine_transition}; + use crate::{ExecutionContext, RuntimeEnv}; use indexmap::IndexMap; use rattler_conda_types::Platform; use rattler_shell::shell::{self, Shell}; @@ -170,6 +292,96 @@ mod tests { ); } + #[test] + fn cmd_switches_between_supported_windows_architectures() { + let script = std::path::Path::new("work/conda_build.bat"); + let runner = native_runner(Platform::Win64); + + let x64_to_arm = ExecutionContext::shared( + RuntimeEnv::for_test(Platform::Win64), + "prefix", + Platform::WinArm64, + Platform::WinArm64, + ); + let arm_command = runner.command_to_run_script(script, &x64_to_arm); + assert_eq!(arm_command.program, "cmd.exe"); + assert_eq!(arm_command.args[..3], ["/d", "/v:on", "/c"]); + assert!(arm_command.args[3].contains("/machine arm64")); + assert!(arm_command.args[3].contains("conda_build.bat")); + assert!(arm_command.args[3].contains("exit /b !ERRORLEVEL!")); + + let spaced_script = std::path::Path::new("work/conda build.bat"); + assert!( + runner + .command_to_run_script(spaced_script, &x64_to_arm) + .args[3] + .contains(r#"cmd.exe /d /c "conda build.bat""#) + ); + + let arm_to_x64 = ExecutionContext::shared( + RuntimeEnv::for_test(Platform::WinArm64), + "prefix", + Platform::Win64, + Platform::Win64, + ); + let x64_command = runner.command_to_run_script(script, &arm_to_x64); + assert!(x64_command.args[3].contains("/machine amd64")); + + let x64_to_x86 = ExecutionContext::shared( + RuntimeEnv::for_test(Platform::Win64), + "prefix", + Platform::Win32, + Platform::Win32, + ); + let x86_command = runner.command_to_run_script(script, &x64_to_x86); + assert!(x86_command.args[3].contains("/machine x86")); + assert!( + x86_command.args[3].contains(r"%SystemRoot%\SysWOW64\cmd.exe"), + "x86 must launch the SysWOW64 command interpreter: {}", + x86_command.args[3] + ); + + let same_arch = ExecutionContext::shared( + RuntimeEnv::for_test(Platform::Win64), + "prefix", + Platform::Win64, + Platform::Win64, + ); + assert_eq!( + runner.command_to_run_script(script, &same_arch).args, + ["/d", "/c", "work/conda_build.bat"] + ); + assert_eq!( + windows_machine_transition(Platform::Win64, Platform::Win32), + Some(WindowsMachine::X86) + ); + assert_eq!( + windows_machine_transition(Platform::Win32, Platform::Win64), + None + ); + assert_eq!( + windows_machine_transition(Platform::Win32, Platform::WinArm64), + None + ); + assert_eq!(WindowsMachine::X86.wow64_processor_architecture(), None); + assert_eq!( + WindowsMachine::Amd64.wow64_processor_architecture(), + Some("AMD64") + ); + assert_eq!( + WindowsMachine::Arm64.wow64_processor_architecture(), + Some("ARM64") + ); + assert_eq!( + windows_machine_transition(Platform::Win32, Platform::Win32), + None + ); + assert_eq!( + windows_machine_transition(Platform::Linux64, Platform::Win32), + None + ); + } + #[test] fn bash_preamble_enables_tracing_after_activation() { let preamble = diff --git a/crates/rattler_build_script/src/runtime.rs b/crates/rattler_build_script/src/runtime.rs index 03eeefc91..84d6e9677 100644 --- a/crates/rattler_build_script/src/runtime.rs +++ b/crates/rattler_build_script/src/runtime.rs @@ -15,7 +15,7 @@ use rattler_conda_types::Platform; #[derive(Debug, Clone)] pub struct RuntimeEnv { env: HashMap, - platform: Platform, + process_platform: Platform, } impl RuntimeEnv { @@ -23,7 +23,7 @@ impl RuntimeEnv { pub fn current() -> Self { Self { env: std::env::vars().collect(), - platform: Platform::current(), + process_platform: Platform::current(), } } @@ -33,13 +33,13 @@ impl RuntimeEnv { pub fn for_test(platform: Platform) -> Self { Self { env: HashMap::new(), - platform, + process_platform: platform, } } - /// The platform rattler-build is running on. - pub fn platform(&self) -> Platform { - self.platform + /// The platform of the rattler-build process. + pub fn process_platform(&self) -> Platform { + self.process_platform } /// Looks up an environment variable by name. @@ -56,7 +56,7 @@ impl RuntimeEnv { /// elsewhere), keyed off the platform rather than the one rattler-build was /// compiled for (unlike [`std::env::consts::EXE_SUFFIX`]). pub(crate) fn exe_suffix(&self) -> &'static str { - if self.platform.is_windows() { + if self.process_platform.is_windows() { ".exe" } else { "" @@ -75,10 +75,10 @@ impl RuntimeEnv { self } - /// Returns a copy that runs on the given platform (builder style, for tests). + /// Returns a copy with the given rattler-build process platform (for tests). #[must_use] - pub fn with_platform(mut self, platform: Platform) -> Self { - self.platform = platform; + pub fn with_process_platform(mut self, platform: Platform) -> Self { + self.process_platform = platform; self } } diff --git a/py-rattler-build/rust/Cargo.lock b/py-rattler-build/rust/Cargo.lock index 9aa387793..236070480 100644 --- a/py-rattler-build/rust/Cargo.lock +++ b/py-rattler-build/rust/Cargo.lock @@ -4957,6 +4957,7 @@ dependencies = [ "tokio-util", "tracing", "which", + "windows-sys 0.61.2", ] [[package]] diff --git a/py-rattler-build/rust/src/render.rs b/py-rattler-build/rust/src/render.rs index efb1e3967..b331f3d6d 100644 --- a/py-rattler-build/rust/src/render.rs +++ b/py-rattler-build/rust/src/render.rs @@ -79,6 +79,7 @@ impl PyRenderConfig { &std::path::PathBuf::new(), &target_platform, &host_platform, + &build_platform, EnvironmentIsolation::default(), &std::path::PathBuf::new(), ) diff --git a/src/lib.rs b/src/lib.rs index 107fe8752..fb3f11322 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -394,6 +394,7 @@ pub async fn get_build_output( &std::path::PathBuf::new(), &build_data.target_platform, &build_data.host_platform, + &build_data.build_platform, build_data.env_isolation, &std::path::PathBuf::new(), ) diff --git a/test-data/recipes/windows-architecture-execution-x86/build.bat b/test-data/recipes/windows-architecture-execution-x86/build.bat new file mode 100644 index 000000000..4b42a2882 --- /dev/null +++ b/test-data/recipes/windows-architecture-execution-x86/build.bat @@ -0,0 +1,12 @@ +@echo off + +echo PROCESSOR_ARCHITECTURE=%PROCESSOR_ARCHITECTURE% +echo PROCESSOR_ARCHITEW6432=%PROCESSOR_ARCHITEW6432% +if /I not "%PROCESSOR_ARCHITECTURE%" == "x86" exit /b 64 +if "%PROCESSOR_ARCHITEW6432%" == "" exit /b 65 + +powershell.exe -NoProfile -Command "$a=[System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture; echo ProcessArchitecture=$a" +if errorlevel 1 exit /b %errorlevel% + +rem This deliberate failure verifies that the outer cmd.exe forwards child status. +exit /b 37 diff --git a/test-data/recipes/windows-architecture-execution-x86/recipe.yaml b/test-data/recipes/windows-architecture-execution-x86/recipe.yaml new file mode 100644 index 000000000..c933e913a --- /dev/null +++ b/test-data/recipes/windows-architecture-execution-x86/recipe.yaml @@ -0,0 +1,6 @@ +package: + name: windows-architecture-execution-x86 + version: 0.1.0 + +build: + script: build.bat diff --git a/test-data/recipes/windows-architecture-execution/build.bat b/test-data/recipes/windows-architecture-execution/build.bat new file mode 100644 index 000000000..8c550d56e --- /dev/null +++ b/test-data/recipes/windows-architecture-execution/build.bat @@ -0,0 +1,12 @@ +@echo off + +echo PROCESSOR_ARCHITECTURE=%PROCESSOR_ARCHITECTURE% +echo PROCESSOR_ARCHITEW6432=%PROCESSOR_ARCHITEW6432% +if /I not "%PROCESSOR_ARCHITECTURE%" == "ARM64" exit /b 64 +if not "%PROCESSOR_ARCHITEW6432%" == "" exit /b 65 + +powershell.exe -NoProfile -Command "$a=[System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture; echo ProcessArchitecture=$a" +if errorlevel 1 exit /b %errorlevel% + +rem This deliberate failure verifies that the outer cmd.exe forwards child status. +exit /b 37 diff --git a/test-data/recipes/windows-architecture-execution/recipe.yaml b/test-data/recipes/windows-architecture-execution/recipe.yaml new file mode 100644 index 000000000..a22534ffc --- /dev/null +++ b/test-data/recipes/windows-architecture-execution/recipe.yaml @@ -0,0 +1,6 @@ +package: + name: windows-architecture-execution + version: 0.1.0 + +build: + script: build.bat diff --git a/test/end-to-end/test_windows_architecture_execution.py b/test/end-to-end/test_windows_architecture_execution.py new file mode 100644 index 000000000..afa3bb2f3 --- /dev/null +++ b/test/end-to-end/test_windows_architecture_execution.py @@ -0,0 +1,83 @@ +import os +from pathlib import Path + +import pytest +from helpers import RattlerBuild + + +def is_windows_arm64_host() -> bool: + """Return whether Windows is natively running on ARM64. + + The E2E runner deliberately uses an emulated x64 rattler-build executable, + so PROCESSOR_ARCHITEW6432 identifies the actual host in that case. + """ + return os.name == "nt" and "ARM64" in { + os.environ.get("PROCESSOR_ARCHITECTURE", "").upper(), + os.environ.get("PROCESSOR_ARCHITEW6432", "").upper(), + } + + +pytestmark = pytest.mark.skipif( + not is_windows_arm64_host(), reason="requires a Windows ARM64 host" +) + + +@pytest.mark.parametrize( + ( + "recipe", + "platform", + "architecture", + "wow64_architecture", + "process_architecture", + ), + [ + ( + "windows-architecture-execution", + "win-arm64", + "ARM64", + "", + "Arm64", + ), + ( + "windows-architecture-execution-x86", + "win-32", + "x86", + "ARM64", + "X86", + ), + ], +) +def test_windows_architecture_execution( + rattler_build: RattlerBuild, + recipes: Path, + tmp_path: Path, + recipe: str, + platform: str, + architecture: str, + wow64_architecture: str, + process_architecture: str, +): + """An emulated x64 rattler-build launches scripts at the requested architecture.""" + result = rattler_build( + "build", + "--recipe", + recipes / recipe, + "--build-platform", + platform, + "--host-platform", + platform, + "--target-platform", + platform, + "--output-dir", + tmp_path / platform, + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + + # The recipes intentionally exit 37 after reporting their architecture. + assert result.returncode != 0 + assert f"PROCESSOR_ARCHITECTURE={architecture}" in output + assert f"PROCESSOR_ARCHITEW6432={wow64_architecture}" in output + assert f"ProcessArchitecture={process_architecture}" in output + assert "Script failed with status 37" in output