diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml new file mode 100644 index 0000000000000..f9aa68fd9b647 --- /dev/null +++ b/.github/workflows/python_wheel_build.yml @@ -0,0 +1,58 @@ +name: 'ROOT Python wheels' + +on: + workflow_dispatch: + inputs: + branch: + description: 'The branch for which the Python wheel has to be generated' + type: string + required: true + default: "experimental-pip-install-root" + push: + branches: experimental-pip-install-root + +jobs: + python_wheel: + runs-on: ubuntu-latest + env: + WORKDIR: /home/runner/work/root + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: Install Kerberos utilities + run: sudo apt-get install -y krb5-user + + - name: Install XRootD client + run: sudo apt-get -y install xrootd-client + + - name: Build wheels + uses: pypa/cibuildwheel@v3.0.1 + with: + output-dir: ${{ env.WORKDIR }}/wheelhouse + + - name: Upload wheel to github + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-test + path: ${{ env.WORKDIR }}/wheelhouse/*.whl + + - name: Upload wheel to EOS + env: + RWEBEOS_KT: ${{ secrets.RWEBEOS_KT }} + KT_FILE_NAME: /tmp/decoded.keytab + EOS_PATH: /eos/project/r/root-eos/www/experimental-python-wheels + EOS_ENDPOINT: root://eosproject-r.cern.ch + KRB5CCNAME: /tmp/krb5cc + working-directory: ${{ env.WORKDIR }} + run: | + echo +++ Content + ls + echo +++ Retrieving the secret + echo ${RWEBEOS_KT} | base64 -d > ${KT_FILE_NAME} + echo +++ Creating the token + kinit -p ${{ secrets.KRB5USER }}@${{ secrets.KRB5REALM }} -kt ${KT_FILE_NAME} + echo +++ Running the copy + xrdcp -rf wheelhouse ${EOS_ENDPOINT}/${EOS_PATH}/ diff --git a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt index 911517294bb6a..47c537c12566c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt +++ b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt @@ -54,6 +54,18 @@ set(libname cppyy) add_library(${libname} SHARED ${headers} ${sources}) # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- libROOTPythonizations.so +# -- ROOT/ +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. +set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") if(MSVC) target_link_libraries(${libname} PUBLIC cppyy_backend Python3::Python) set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) diff --git a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt index f296f3886db91..9760862b7cfef 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt +++ b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt @@ -31,6 +31,18 @@ if(MSVC) endif() # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- libROOTPythonizations.so +# -- ROOT/ +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. +set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") target_link_libraries(${libname} Core ${CMAKE_DL_LIBS}) # cppyy uses ROOT headers from binary directory diff --git a/bindings/pyroot/pythonizations/CMakeLists.txt b/bindings/pyroot/pythonizations/CMakeLists.txt index 9a16d7d2887ca..5f49ed2ceaeb1 100644 --- a/bindings/pyroot/pythonizations/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/CMakeLists.txt @@ -174,7 +174,7 @@ add_dependencies(${libname} ROOTPythonizationsPySources) # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES_NO_VERSION}) -target_link_libraries(${libname} PUBLIC Core Tree cppyy) +target_link_libraries(${libname} PUBLIC ROOT::Core ROOT::Tree cppyy) if(MSVC) set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) set_target_properties(${libname} PROPERTIES SUFFIX ".pyd") @@ -228,6 +228,18 @@ else() LIBRARY_OUTPUT_DIRECTORY ${pymoduledir_build}) endif() +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- ROOT/ +# -- -- libROOTPythonizations.so +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. + if(NOT MSVC) # Make sure that relative RUNPATH to main ROOT libraries is always correct. @@ -242,7 +254,8 @@ if(NOT MSVC) else() set_target_properties(${libname} PROPERTIES BUILD_RPATH "$ORIGIN/${pymoduledir_to_libdir_build}" - INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}" + # ATTENTION: NEXT PART IS CRUCIAL SEE COMMENT ABOVE + INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}:${CMAKE_INSTALL_RPATH}:$ORIGIN/lib" ) endif() diff --git a/bindings/pyroot/pythonizations/python/ROOT/__init__.py b/bindings/pyroot/pythonizations/python/ROOT/__init__.py index 0cdf689ca2166..d12a7058795d6 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/__init__.py +++ b/bindings/pyroot/pythonizations/python/ROOT/__init__.py @@ -8,9 +8,27 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ -import importlib +# First, check for presence of important dependencies import os +import shlex import sys +import subprocess +import textwrap + +try: + # Mimic what cling does to find standard headers include path + cmd = shlex.split('c++ -xc++ -E -v /dev/null') + env = os.environ + env.update({'LC_ALL': 'C'}) + subprocess.run(cmd, env=env, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +except Exception as e: + msg = ( + "Could not find a C++ compiler when importing ROOT. Make sure a C++ compiler as well as the C++ standard " + "libraries are installed. For example, run `[apt,dnf] install g++` or follow similar instructions for your " + "distribution. For more info, visit https://root.cern/install/dependencies" + ) + raise ImportError(textwrap.fill(msg, width=80)) from e # Prevent cppyy's check for the PCH os.environ["CLING_STANDARD_PCH"] = "none" diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index 3218ba04ff99d..43fd05af219ec 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -11,6 +11,12 @@ from ._numbadeclare import _NumbaDeclareDecorator from ._pythonization import pythonization +# Because the facade acts as the actual ROOT package for `import` purposes, +# we have to expose any further scripts that must be picked up by the pip +# build backend so that the generated executable can properly run +# `from ROOT._name_of_script import main`. See pyproject.toml for the list +# of enabled scripts. +from . import _rootcli as _rootexe class PyROOTConfiguration(object): """Class for configuring PyROOT""" @@ -56,6 +62,8 @@ def _subimport(name): class ROOTFacade(types.ModuleType): """Facade class for ROOT module""" + _rootcli = _rootexe + def __init__(self, module, is_ipython): types.ModuleType.__init__(self, module.__name__) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py b/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py new file mode 100644 index 0000000000000..25bd1b5e3e33c --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py @@ -0,0 +1,25 @@ +import os +import subprocess +import sys + +ROOT_HOME = os.path.dirname(__file__) + + +def main() -> None: + # Ensure all ROOT libraries are found at runtime by the process + os.environ['LD_LIBRARY_PATH'] = os.path.join(ROOT_HOME, 'lib') + # Finds the ROOT process from the current installation directory + rootexe = os.path.join(ROOT_HOME, 'bin', 'root.exe') + if not os.path.exists(rootexe): + msg = (f"Could not find 'root' executable in directory '{os.path.join(ROOT_HOME, "bin")}'. " + "Something is wrong in the ROOT installation.") + raise FileNotFoundError(msg) + # Make sure command line arguments are preserved + args = [rootexe] + sys.argv[1:] + # Run the actual ROOT executable and return the exit code to the main Python process + out = subprocess.run(args) + return out.returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 5a7e7b47f50cc..fe1d093221577 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -677,7 +677,9 @@ set(Python3_FIND_FRAMEWORK LAST) # Even if we don't build PyROOT, one still need python executable to run some scripts list(APPEND python_components Interpreter) if(pyroot OR tmva-pymva) - list(APPEND python_components Development) + # We have to only look for the Python development module in order to be able to build ROOT with a pip backend + # In particular, it is forbidden to link against libPython.so, see https://peps.python.org/pep-0513/#libpythonx-y-so-1 + list(APPEND python_components Development.Module) endif() if(tmva-pymva) list(APPEND python_components NumPy) @@ -1759,7 +1761,7 @@ endif(tmva) #---Check for PyROOT--------------------------------------------------------------------- if(pyroot) - if(Python3_Development_FOUND) + if(Python3_Development.Module_FOUND) message(STATUS "PyROOT: development package found. Building for version ${Python3_VERSION}") else() if(fail-on-missing) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000..6e62f1331c0ea --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["cmake", "setuptools<72", "wheel", "ninja", "numpy"] + +[project] +name = "ROOT" +version = "0.1a7" + +requires-python = ">=3.8" +authors = [ + {name = "Vincenzo Eduardo Padulano", email="vincenzo.eduardo.padulano@cern.ch"} +] +maintainers = [ + {name = "Vincenzo Eduardo Padulano", email = "vincenzo.eduardo.padulano@cern.ch"} +] +description = "An open-source data analysis framework used by high energy physics and others." +readme = "README.md" +license = {file = "LICENSE"} + +[project.scripts] +root = "ROOT._rootcli:main" + +[tool.cibuildwheel] +# Increase pip debugging output +build-verbosity = 1 +#build="cp3{8,9,10,11,12,13}-manylinux_x86_64" +build="cp313-manylinux_x86_64" +manylinux-x86_64-image = "manylinux_2_28" + +# Install system libraries +[tool.cibuildwheel.linux] +before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel libjpeg-devel giflib-devel" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000..33b66144d20b9 --- /dev/null +++ b/setup.py @@ -0,0 +1,128 @@ +""" +setuptools-based build of ROOT. + +This script uses setuptools API to steer a custom CMake build of ROOT. All build +configuration options are specified in the class responsible for building. The +installation step produces the following output target install directory: + +``` +site-packages/ +-- cppyy/ +-- cppyy_backend/ +-- libcppyy.so +-- libcppyy_backend.so +-- ROOT/ +-- -- libROOTPythonizations.so +``` + +A custom extension module is injected in the setuptools setup to properly +generate the wheel with CPython extension metadata. +""" + +from setuptools import setup, find_packages, Extension +import pathlib +import tempfile +from setuptools.command.build import build as _build +from setuptools.command.install import install as _install + +import subprocess +import os +import shlex + +# Get the long description from the README file +SOURCE_DIR = pathlib.Path(__file__).parent.resolve() +LONG_DESCRIPTION = (SOURCE_DIR / "README.md").read_text(encoding="utf-8") + +BUILD_DIR = tempfile.mkdtemp() +INSTALL_DIR = tempfile.mkdtemp() + + +class ROOTBuild(_build): + def run(self): + _build.run(self) + + # Configure ROOT build + base_opts = shlex.split("cmake -GNinja -Dccache=ON") + mode_opts = shlex.split( + "-Dbuiltin_nlohmannjson=ON -Dbuiltin_tbb=ON -Dbuiltin_xrootd=ON " # builtins + "-Dbuiltin_lz4=ON -Dbuiltin_lzma=ON -Dbuiltin_zstd=ON -Dbuiltin_xxhash=ON" # builtins + "-Druntime_cxxmodules=ON -Drpath=ON -Dfail-on-missing=ON " # Generic build configuration + "-Dgminimal=ON -Dasimage=ON -Dopengl=OFF " # Graphics + "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dssl=ON -Dimt=ON " + "-Droofit=ON" + ) + dirs_opts = shlex.split(f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") + configure_command = base_opts + mode_opts + dirs_opts + print(f"\n\n{' '.join(configure_command)}\n\n") + subprocess.run(configure_command, check=True) + + # Run build with CMake + build_command = f"cmake --build {BUILD_DIR} -j{os.cpu_count()}" + subprocess.run(shlex.split(build_command), check=True) + + +class ROOTInstall(_install): + def _get_install_path(self): + if hasattr(self, "bdist_dir") and self.bdist_dir: + install_path = self.bdist_dir + else: + install_path = self.install_lib + + return install_path + + def run(self): + _install.run(self) + + install_cmd = f"cmake --build {BUILD_DIR} --target install" + subprocess.run(shlex.split(install_cmd), check=True) + + install_path = self._get_install_path() + + lib_dir = os.path.join(INSTALL_DIR, "lib") + + # Copy ROOT installation tree to the ROOT package directory in the pip installation path + self.copy_tree(INSTALL_DIR, os.path.join(install_path, "ROOT")) + + # Copy cppyy packages separately + self.copy_tree(os.path.join(lib_dir, "cppyy"), os.path.join(install_path, "cppyy")) + self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), os.path.join(install_path, "cppyy_backend")) + + # Finally copy CPython extension libraries + self.copy_file(os.path.join(lib_dir, "libcppyy.so"), install_path) + self.copy_file(os.path.join(lib_dir, "libcppyy_backend.so"), install_path) + self.copy_file(os.path.join(lib_dir, "ROOT", "libROOTPythonizations.so"), os.path.join(install_path, "ROOT")) + + def get_outputs(self): + outputs = _install.get_outputs(self) + return outputs + + +class DummyExtension(Extension): + """ + Dummy CPython extension for setuptools setup. + + In order to generate the wheel with CPython extension metadata (i.e. + producing one wheel per supported Python version), setuptools requires that + at least one CPython extension is declared in the `ext_modules` kwarg passed + to the `setup` function. Usually, declaring a CPython extension triggers + compilation of the corresponding sources, but in this case we already do + that in the CMake build step. This class defines a dummy extension that + can be declared to setuptools while avoiding any further compilation step. + """ + + def __init__(_): + super().__init__(name="Dummy", sources=[]) + + +pkgs = find_packages("bindings/pyroot/pythonizations/python") + find_packages( + "bindings/pyroot/cppyy/cppyy/python", include=["cppyy"] +) + +s = setup( + long_description=LONG_DESCRIPTION, + package_dir={"": "bindings/pyroot/pythonizations/python", "cppyy": "bindings/pyroot/cppyy/cppyy/python"}, + packages=pkgs, + # Crucial to signal this is not a pure Python package + ext_modules=[DummyExtension()], + cmdclass={"build": ROOTBuild, "install": ROOTInstall}, +)