diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 807fb41460..109eef57e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -291,7 +291,7 @@ jobs: needs: changes if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true' - runs-on: windows-2022 + runs-on: ${{ matrix.runs-on && matrix.runs-on || 'windows-2022' }} strategy: fail-fast: false matrix: @@ -310,11 +310,16 @@ jobs: - python-version: '3.14' conda-version: canary test-type: parallel + - python-version: '3.14' + conda-version: release + test-type: custom + pytest-expression: test_build_command_win_arm64_wrapper or test_win_arm64_build_on_emulated_win_64 + runs-on: windows-11-arm env: ErrorActionPreference: Stop # powershell exit on first error CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }} # Exclude benchmark and slow tests from Windows parallel runs - PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial and not benchmark' || 'not serial and not slow and not benchmark' }} + PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial and not benchmark' || (matrix.test-type == 'parallel' && 'not serial and not slow and not benchmark' || '') }} steps: - name: Checkout Source @@ -343,10 +348,15 @@ jobs: with: condarc-file: .github\condarc run-post: false # skip post cleanup - pkgs-dirs: D:\conda_pkgs_dir - installation-dir: D:\conda + pkgs-dirs: ${{ runner.arch == 'ARM64' && 'C' || 'D'}}:\conda_pkgs_dir + installation-dir: ${{ runner.arch == 'ARM64' && 'C' || 'D'}}:\conda + # force x64 even on ARM64 to test emulation + architecture: x64 + env: + CONDA_SUBDIR: win-64 - name: Choco Install + if: runner.arch != 'ARM64' # We may need the complete tooling so that cmake tests pass on windows run: choco install visualstudio2017-workload-vctools @@ -380,7 +390,8 @@ jobs: --basetemp=${{ runner.temp }} --durations-path=durations\${{ runner.os }}.json -n auto - -m "${{ env.PYTEST_MARKER }}" + ${{ env.PYTEST_MARKER && '-m "' || ''}}${{ env.PYTEST_MARKER }}${{ env.PYTEST_MARKER && '"' || ''}} + ${{ matrix.pytest-expression && '-k "' || ''}}${{ matrix.pytest-expression }}${{ matrix.pytest-expression && '"' || ''}} - name: Upload Coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/conda_build/_rattler_build/compat.py b/conda_build/_rattler_build/compat.py index 3ead49e3c2..f0de34fa83 100644 --- a/conda_build/_rattler_build/compat.py +++ b/conda_build/_rattler_build/compat.py @@ -439,8 +439,8 @@ def run_rattler( for variant in config_files: variant_config = variant_config.merge(VariantConfig.from_file(variant)) - def get_config_value(name): - value = variant_config.get(name, config.subdir) + def get_config_value(name, fallback=None): + value = variant_config.get(name, fallback) if isinstance(value, list): if len(value) != 1: @@ -450,9 +450,9 @@ def get_config_value(name): return value[0] return value - build_platform = get_config_value("build_platform") - host_platform = get_config_value("host_platform") - target_platform = get_config_value("target_platform") + build_platform = config.build_subdir # Overridable via CONDA_SUBDIR env var + host_platform = get_config_value("target_platform", config.host_subdir) + target_platform = get_config_value("target_platform", config.target_subdir) noarch_build_platform = get_config_value("noarch_build_platform") # common tool / platform / render configuration diff --git a/conda_build/windows.py b/conda_build/windows.py index f9234f9797..4302b39098 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -1,12 +1,23 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + import os +import platform import pprint +import sys +import sysconfig +from functools import cache from itertools import product from os.path import dirname, isdir, isfile, join +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Literal # importing setuptools patches distutils so that it knows how to find VC for python 2.7 import setuptools # noqa +from conda.base.context import context # Leverage the hard work done by setuptools/distutils to find vcvarsall using # either the registry or the VS**COMNTOOLS environment variable @@ -74,6 +85,38 @@ def fix_staged_scripts(scripts_dir, config): os.remove(join(scripts_dir, fn)) +@cache +def get_native_windows_architecture() -> Literal["AMD64", "ARM64", "x86"] | str | None: + """ + Python 3.12+ `platform.machine()` on Windows reports the actual machine, + not the running interpreter. So we can just use that. + + For prior versions, query the registry. + + Returns None if it value cannot be obtained. + """ + if sys.platform != "win32": + raise OSError("This function is only supported on Windows.") + + if sys.version_info >= (3, 12): + return platform.machine() or None + + import winreg + + registry_path = r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment" + + try: + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, registry_path, 0, winreg.KEY_READ + ) as key: + native_arch, _ = winreg.QueryValueEx(key, "PROCESSOR_ARCHITECTURE") + return native_arch.upper() + except Exception as e: + log = get_logger(__name__) + log.debug("Could not detect native architecture via registry", exc_info=e) + return None + + def build_vcvarsall_vs_path(version): """ Given the Visual Studio version, returns the default path to the @@ -293,6 +336,13 @@ def write_build_scripts(m, env, bld_bat): env["PYTHONDONTWRITEBYTECODE"] = True import codecs + if m.config.build_subdir != _running_subdir(): + # Can't trust the parent environment under these conditions + build_arch = _build_arch(m) + env["PROCESSOR_ARCHITECTURE"] = build_arch + if build_arch == get_native_windows_architecture(): + env.pop("PROCESSOR_ARCHITEW6432", None) + with codecs.getwriter("utf-8")(open(env_script, "wb")) as fo: # more debuggable with echo on fo.write("@echo on\n") @@ -347,6 +397,52 @@ def write_build_scripts(m, env, bld_bat): return work_script, env_script +def _build_arch(m) -> Literal["AMD64", "ARM64", "x86"]: + """ + If conda-build is run from e.g. a win-64 environment on a win-arm64 machine + users may want to build natively by setting build_platform="win-arm64". + In those cases, we need to ensure that the CMD process is native ARM64 + via this `start` wrapper. Otherwise Windows picks the AMD64 slice! + This gives you the adequate /machine flag value for `start`. + """ + build_arch = m.config.build_subdir.split("-")[1].upper() + return {"64": "AMD64", "32": "x86"}.get(build_arch, build_arch) + + +def _running_subdir(): + """ + On Python 3.12+, `platform.machine()` (on which conda.base.context._native_subdir() relies) + may report ARM64 even when running from a win-64 installation. + This is different from what one can observe on macOS/Rosetta, where an emulated osx-64 Python + on Apple Silicon reports x86_64. + + More context at https://github.com/python/cpython/issues/98962. + + The most obvious way is to check %PROCESSOR_ARCHITECTURE%, if available, but we need to hope + it was not overridden. `sysconfig.get_platform()` seems to be accurate enough. + """ + if os.name == "nt": + arch = ( + sysconfig.get_platform().lower() + ) # -> Literal['win-amd64', 'win-arm64', 'win32'] + return {"win-amd64": "win-64", "win32": "win-32"}.get(arch, arch) + return context._native_subdir() + + +def build_command_arguments(m, script: str) -> list[str]: + if m.config.build_subdir != _running_subdir(): + # See docstring of _cmd_machine_flag() + wrapper = os.path.join(os.path.dirname(script), "_conda_build_wrapper.bat") + with open(wrapper, "w") as f: + f.write( + "@echo off\r\n" + f'start /b /wait /machine {_build_arch(m)} cmd.exe /d /c "{script}"\r\n' + "exit /b %ERRORLEVEL%\r\n" + ) + return ["cmd.exe", "/d", "/c", os.path.basename(wrapper)] + return ["cmd.exe", "/d", "/c", os.path.basename(script)] + + def build(m, bld_bat, stats, provision_only=False): # TODO: Prepending the prefixes here should probably be guarded by # if not m.activate_build_script: @@ -390,7 +486,7 @@ def build(m, bld_bat, stats, provision_only=False): work_script, env_script = write_build_scripts(m, env, bld_bat) if not provision_only and os.path.isfile(work_script): - cmd = ["cmd.exe", "/d", "/c", os.path.basename(work_script)] + cmd = build_command_arguments(m, work_script) # rewrite long paths in stdout back to their env variables if m.config.debug or m.config.no_rewrite_stdout_env: rewrite_env = None diff --git a/docs/source/resources/define-metadata.rst b/docs/source/resources/define-metadata.rst index d1735fa2e0..fa4c4bc20a 100644 --- a/docs/source/resources/define-metadata.rst +++ b/docs/source/resources/define-metadata.rst @@ -2106,7 +2106,7 @@ variables are booleans. - The NumPy version as an integer such as ``111``. See the CONDA_NPY :ref:`environment variable `. * - build_platform - - The native subdir of the conda executable + - The native subdir of the conda executable. Override with ``CONDA_SUBDIR`` env var when calling ``conda-build``. The use of the Python version selectors, `py27`, `py34`, etc. is discouraged in favor of the more general comparison operators. Additional selectors in this diff --git a/docs/source/user-guide/environment-variables.rst b/docs/source/user-guide/environment-variables.rst index 1689b9106b..849dba7f66 100644 --- a/docs/source/user-guide/environment-variables.rst +++ b/docs/source/user-guide/environment-variables.rst @@ -130,7 +130,7 @@ inherited from the shell environment in which you invoke * - STDLIB_DIR - Python standard library location. * - build_platform - - The native subdir of the conda executable + - The native subdir of the conda executable. Override with ``CONDA_SUBDIR`` env var when calling ``conda-build``. Unix-style packages on Windows, which are usually statically linked to executables, are built in a special ``Library`` diff --git a/news/6047-win-arm64-native.md b/news/6047-win-arm64-native.md new file mode 100644 index 0000000000..c260e274af --- /dev/null +++ b/news/6047-win-arm64-native.md @@ -0,0 +1,20 @@ +### Enhancements + +* + +### Bug fixes + +* Make CMD subprocesses match `build_platform` architecture when running emulated Python processes. (#6048 via #6047) +* `recipe.yaml` build/host platform handling now behaves in the same as it did in `meta.yaml`. Accidentally allowed `{build,host}_platform` settings in `conda_build_config.yaml` are no longer valid; instead, users can export `CONDA_SUBDIR` and define the `target_platform` setting, respectively. (#6047) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_build.py b/tests/test_build.py index 8e7f74c986..a3a4fef273 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -10,6 +10,7 @@ import json import logging import os +import subprocess import sys from collections import defaultdict from contextlib import nullcontext @@ -19,7 +20,7 @@ import pytest from conda.common.compat import on_win -from conda_build import api, build +from conda_build import api, build, windows from conda_build.exceptions import CondaBuildUserError from conda_build.metadata import MetaData from conda_build.variants import get_default_variant @@ -434,3 +435,83 @@ def _minimal_meta_with_reqs(requirements: dict) -> defaultdict: build._warn_implicit_numpy_variant(m) n_warn = sum(1 for r in caplog.records if r.levelno == logging.WARNING) assert n_warn == (1 if expected_warning else 0) + + +@pytest.mark.parametrize( + "build_subdir,running_subdir,wrapped", + [ + ("win-arm64", "win-64", True), + ("win-64", "win-arm64", True), + ("win-arm64", "win-arm64", False), + ("win-64", "win-64", False), + ], +) +def test_build_command_win_arm64_wrapper( + testing_metadata: MetaData, + monkeypatch, + tmp_path, + build_subdir, + running_subdir, + wrapped, +): + platform, arch = build_subdir.split("-") + testing_metadata.config.platform = platform + testing_metadata.config.arch = arch + testing_metadata.config.variant["target_platform"] = build_subdir + monkeypatch.setattr(windows, "_running_subdir", lambda: running_subdir) + + work_script = tmp_path / "conda_build.bat" + work_script.write_text("@echo off\r\n") + wrapper = tmp_path / "_conda_build_wrapper.bat" + + cmd = windows.build_command_arguments(testing_metadata, str(work_script)) + + if not wrapped: + assert cmd == ["cmd.exe", "/d", "/c", "conda_build.bat"] + assert not wrapper.exists() + return + + assert cmd == ["cmd.exe", "/d", "/c", "_conda_build_wrapper.bat"] + + contents = wrapper.read_text() + contents_bytes = wrapper.read_bytes() + machine = "AMD64" if build_subdir == "win-64" else "ARM64" + assert f"/machine {machine}" in contents + assert "/b" in contents + assert "/wait" in contents + assert "%ERRORLEVEL%" in contents + assert str(work_script) in contents + # batch files must be CRLF; a bare LF breaks cmd.exe + assert contents_bytes.count(b"\n") == contents_bytes.count(b"\r\n") + + +@pytest.mark.skipif( + not (on_win and windows.get_native_windows_architecture() == "ARM64"), + reason="Windows ARM only test", +) +def test_win_arm64_build_on_emulated_win_64( + testing_metadata: MetaData, tmp_path, capsys +): + """ + This test checks the architecture of the launched CMD on Windows ARM + machines that are running emulated x64 processes. Critical for bootstrapping + win-arm64 distributions from their win-64 counterparts via emulation. + """ + cmdlet = "[System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture" + (tmp_path / "bld.bat").write_text( + f"echo PROCESSOR_ARCHITECTURE=%PROCESSOR_ARCHITECTURE%\r\n" + f"powershell -Command \"'ProcessArchitecture=' + {cmdlet}\"\r\n" + f"exit /b 42\r\n" + ) + testing_metadata.config.arch = "arm64" # this is for build_platform + testing_metadata.config.variant["target_platform"] = "win-arm64" + with pytest.raises(subprocess.CalledProcessError) as exc: + windows.build(testing_metadata, str(tmp_path / "bld.bat"), {}) + assert exc.value.returncode == 42 + out, err = capsys.readouterr() + print(out) + print("---") + print("Directory contents:") + print(*sorted(os.listdir(testing_metadata.config.work_dir)), sep="\n") + assert "PROCESSOR_ARCHITECTURE=ARM64" in out + assert "ProcessArchitecture=Arm64" in out