Test Python wheel build github workflow

.github/workflows/python_wheel_build.yml

@@ -0,0 +1,58 @@
+name: 'ROOT Python wheels'
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'The branch for which the Python wheel has to be generated'
+        type: string
+        required: true
+        default: "experimental-pip-install-root"
+  push:
+    branches: experimental-pip-install-root
+
+jobs:
+  python_wheel:
+    runs-on: ubuntu-latest
+    env:
+      WORKDIR: /home/runner/work/root
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
+      - name: Install Kerberos utilities
+        run: sudo apt-get install -y krb5-user
+
+      - name: Install XRootD client
+        run: sudo apt-get -y install xrootd-client
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.0.1
+        with:
+          output-dir: ${{ env.WORKDIR }}/wheelhouse
+
+      - name: Upload wheel to github
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-test
+          path: ${{ env.WORKDIR }}/wheelhouse/*.whl
+
+      - name: Upload wheel to EOS
+        env:
+          RWEBEOS_KT: ${{ secrets.RWEBEOS_KT }}
+          KT_FILE_NAME: /tmp/decoded.keytab
+          EOS_PATH: /eos/project/r/root-eos/www/experimental-python-wheels
+          EOS_ENDPOINT: root://eosproject-r.cern.ch
+          KRB5CCNAME: /tmp/krb5cc
+        working-directory: ${{ env.WORKDIR }}
+        run: |
+          echo +++ Content
+          ls
+          echo +++ Retrieving the secret
+          echo ${RWEBEOS_KT} | base64 -d > ${KT_FILE_NAME}
+          echo +++ Creating the token
+          kinit  -p ${{ secrets.KRB5USER }}@${{ secrets.KRB5REALM }} -kt ${KT_FILE_NAME}
+          echo +++ Running the copy
+          xrdcp -rf wheelhouse ${EOS_ENDPOINT}/${EOS_PATH}/

bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt

@@ -54,6 +54,18 @@ set(libname cppyy)
 add_library(${libname} SHARED ${headers} ${sources})
 # Set the suffix to '.so' and the prefix to 'lib'
 set_target_properties(${libname} PROPERTIES  ${ROOT_LIBRARY_PROPERTIES})
+# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends.
+# After the `pip install` step, the target install user directory will look like:
+# site-packages/
+# -- cppyy/
+# -- cppyy_backend/
+# -- libcppyy.so
+# -- libcppyy_backend.so
+# -- libROOTPythonizations.so
+# -- ROOT/
+# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension
+# shared libraries.
+set_target_properties(${libname} PROPERTIES  INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib")
 if(MSVC)
   target_link_libraries(${libname} PUBLIC cppyy_backend Python3::Python)
   set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE)

bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt

@@ -31,6 +31,18 @@ if(MSVC)
 endif()
 # Set the suffix to '.so' and the prefix to 'lib'
 set_target_properties(${libname} PROPERTIES  ${ROOT_LIBRARY_PROPERTIES})
+# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends.
+# After the `pip install` step, the target install user directory will look like:
+# site-packages/
+# -- cppyy/
+# -- cppyy_backend/
+# -- libcppyy.so
+# -- libcppyy_backend.so
+# -- libROOTPythonizations.so
+# -- ROOT/
+# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension
+# shared libraries.
+set_target_properties(${libname} PROPERTIES  INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib")
 target_link_libraries(${libname} Core ${CMAKE_DL_LIBS})
 
 # cppyy uses ROOT headers from binary directory

bindings/pyroot/pythonizations/CMakeLists.txt

@@ -174,7 +174,7 @@ add_dependencies(${libname} ROOTPythonizationsPySources)
 
 # Set the suffix to '.so' and the prefix to 'lib'
 set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES_NO_VERSION})
-target_link_libraries(${libname} PUBLIC Core Tree cppyy)
+target_link_libraries(${libname} PUBLIC ROOT::Core ROOT::Tree cppyy)
 if(MSVC)
   set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
   set_target_properties(${libname} PROPERTIES SUFFIX ".pyd")
@@ -228,6 +228,18 @@ else()
       LIBRARY_OUTPUT_DIRECTORY ${pymoduledir_build})
 endif()
 
+# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends.
+# After the `pip install` step, the target install user directory will look like:
+# site-packages/
+# -- cppyy/
+# -- cppyy_backend/
+# -- libcppyy.so
+# -- libcppyy_backend.so
+# -- ROOT/
+# -- -- libROOTPythonizations.so
+# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension
+# shared libraries.
+
 if(NOT MSVC)
   # Make sure that relative RUNPATH to main ROOT libraries is always correct.
 
@@ -242,7 +254,8 @@ if(NOT MSVC)
   else()
     set_target_properties(${libname} PROPERTIES
         BUILD_RPATH "$ORIGIN/${pymoduledir_to_libdir_build}"
-        INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}"
+        # ATTENTION: NEXT PART IS CRUCIAL SEE COMMENT ABOVE
+        INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}:${CMAKE_INSTALL_RPATH}:$ORIGIN/lib"
     )
   endif()
 

bindings/pyroot/pythonizations/python/ROOT/__init__.py

@@ -8,9 +8,27 @@
 # For the list of contributors see $ROOTSYS/README/CREDITS.                    #
 ################################################################################
 
-import importlib
+# First, check for presence of important dependencies
 import os
+import shlex
 import sys
+import subprocess
+import textwrap
+
+try:
+    # Mimic what cling does to find standard headers include path
+    cmd = shlex.split('c++ -xc++ -E -v /dev/null')
+    env = os.environ
+    env.update({'LC_ALL': 'C'})
+    subprocess.run(cmd, env=env, check=True,
+                   stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+except Exception as e:
+    msg = (
+        "Could not find a C++ compiler when importing ROOT. Make sure a C++ compiler as well as the C++ standard "
+        "libraries are installed. For example, run `[apt,dnf] install g++` or follow similar instructions for your "
+        "distribution. For more info, visit https://root.cern/install/dependencies"
+    )
+    raise ImportError(textwrap.fill(msg, width=80)) from e
 
 # Prevent cppyy's check for the PCH
 os.environ["CLING_STANDARD_PCH"] = "none"

bindings/pyroot/pythonizations/python/ROOT/_facade.py

@@ -11,6 +11,12 @@
 from ._numbadeclare import _NumbaDeclareDecorator
 from ._pythonization import pythonization
 
+# Because the facade acts as the actual ROOT package for `import` purposes,
+# we have to expose any further scripts that must be picked up by the pip
+# build backend so that the generated executable can properly run
+# `from ROOT._name_of_script import main`. See pyproject.toml for the list
+# of enabled scripts.
+from . import _rootcli as _rootexe
 
 class PyROOTConfiguration(object):
     """Class for configuring PyROOT"""
@@ -56,6 +62,8 @@ def _subimport(name):
 class ROOTFacade(types.ModuleType):
     """Facade class for ROOT module"""
 
+    _rootcli = _rootexe
+
     def __init__(self, module, is_ipython):
         types.ModuleType.__init__(self, module.__name__)
 

bindings/pyroot/pythonizations/python/ROOT/_rootcli.py

@@ -0,0 +1,25 @@
+import os
+import subprocess
+import sys
+
+ROOT_HOME = os.path.dirname(__file__)
+
+
+def main() -> None:
+    # Ensure all ROOT libraries are found at runtime by the process
+    os.environ['LD_LIBRARY_PATH'] = os.path.join(ROOT_HOME, 'lib')
+    # Finds the ROOT process from the current installation directory
+    rootexe = os.path.join(ROOT_HOME, 'bin', 'root.exe')
+    if not os.path.exists(rootexe):
+        msg = (f"Could not find 'root' executable in directory '{os.path.join(ROOT_HOME, "bin")}'. "
+               "Something is wrong in the ROOT installation.")
+        raise FileNotFoundError(msg)
+    # Make sure command line arguments are preserved
+    args = [rootexe] + sys.argv[1:]
+    # Run the actual ROOT executable and return the exit code to the main Python process
+    out = subprocess.run(args)
+    return out.returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cmake/modules/SearchInstalledSoftware.cmake

@@ -677,7 +677,9 @@ set(Python3_FIND_FRAMEWORK LAST)
 # Even if we don't build PyROOT, one still need python executable to run some scripts
 list(APPEND python_components Interpreter)
 if(pyroot OR tmva-pymva)
-  list(APPEND python_components Development)
+  # We have to only look for the Python development module in order to be able to build ROOT with a pip backend
+  # In particular, it is forbidden to link against libPython.so, see https://peps.python.org/pep-0513/#libpythonx-y-so-1
+  list(APPEND python_components Development.Module)
 endif()
 if(tmva-pymva)
   list(APPEND python_components NumPy)
@@ -1759,7 +1761,7 @@ endif(tmva)
 #---Check for PyROOT---------------------------------------------------------------------
 if(pyroot)
 
-  if(Python3_Development_FOUND)
+  if(Python3_Development.Module_FOUND)
     message(STATUS "PyROOT: development package found. Building for version ${Python3_VERSION}")
   else()
     if(fail-on-missing)

pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["cmake", "setuptools<72", "wheel", "ninja", "numpy"]
+
+[project]
+name = "ROOT"
+version = "0.1a7"
+
+requires-python = ">=3.8"
+authors = [
+  {name = "Vincenzo Eduardo Padulano", email="vincenzo.eduardo.padulano@cern.ch"}
+]
+maintainers = [
+  {name = "Vincenzo Eduardo Padulano", email = "vincenzo.eduardo.padulano@cern.ch"}
+]
+description = "An open-source data analysis framework used by high energy physics and others."
+readme = "README.md"
+license = {file = "LICENSE"}
+
+[project.scripts]
+root = "ROOT._rootcli:main"
+
+[tool.cibuildwheel]
+# Increase pip debugging output
+build-verbosity = 1
+#build="cp3{8,9,10,11,12,13}-manylinux_x86_64"
+build="cp313-manylinux_x86_64"
+manylinux-x86_64-image = "manylinux_2_28"
+
+# Install system libraries
+[tool.cibuildwheel.linux]
+before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel libjpeg-devel giflib-devel"

setup.py

@@ -0,0 +1,128 @@
+"""
+setuptools-based build of ROOT.
+
+This script uses setuptools API to steer a custom CMake build of ROOT. All build
+configuration options are specified in the class responsible for building. The
+installation step produces the following output target install directory:
+
+```
+site-packages/
+-- cppyy/
+-- cppyy_backend/
+-- libcppyy.so
+-- libcppyy_backend.so
+-- ROOT/
+-- -- libROOTPythonizations.so
+```
+
+A custom extension module is injected in the setuptools setup to properly
+generate the wheel with CPython extension metadata.
+"""
+
+from setuptools import setup, find_packages, Extension
+import pathlib
+import tempfile
+from setuptools.command.build import build as _build
+from setuptools.command.install import install as _install
+
+import subprocess
+import os
+import shlex
+
+# Get the long description from the README file
+SOURCE_DIR = pathlib.Path(__file__).parent.resolve()
+LONG_DESCRIPTION = (SOURCE_DIR / "README.md").read_text(encoding="utf-8")
+
+BUILD_DIR = tempfile.mkdtemp()
+INSTALL_DIR = tempfile.mkdtemp()
+
+
+class ROOTBuild(_build):
+    def run(self):
+        _build.run(self)
+
+        # Configure ROOT build
+        base_opts = shlex.split("cmake -GNinja -Dccache=ON")
+        mode_opts = shlex.split(
+            "-Dbuiltin_nlohmannjson=ON -Dbuiltin_tbb=ON -Dbuiltin_xrootd=ON "  # builtins
+            "-Dbuiltin_lz4=ON -Dbuiltin_lzma=ON -Dbuiltin_zstd=ON -Dbuiltin_xxhash=ON"  # builtins
+            "-Druntime_cxxmodules=ON -Drpath=ON -Dfail-on-missing=ON "  # Generic build configuration
+            "-Dgminimal=ON -Dasimage=ON -Dopengl=OFF "  # Graphics
+            "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dssl=ON -Dimt=ON "
+            "-Droofit=ON"
+        )
+        dirs_opts = shlex.split(f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}")
+        configure_command = base_opts + mode_opts + dirs_opts
+        print(f"\n\n{' '.join(configure_command)}\n\n")
+        subprocess.run(configure_command, check=True)
+
+        # Run build with CMake
+        build_command = f"cmake --build {BUILD_DIR} -j{os.cpu_count()}"
+        subprocess.run(shlex.split(build_command), check=True)
+
+
+class ROOTInstall(_install):
+    def _get_install_path(self):
+        if hasattr(self, "bdist_dir") and self.bdist_dir:
+            install_path = self.bdist_dir
+        else:
+            install_path = self.install_lib
+
+        return install_path
+
+    def run(self):
+        _install.run(self)
+
+        install_cmd = f"cmake --build {BUILD_DIR} --target install"
+        subprocess.run(shlex.split(install_cmd), check=True)
+
+        install_path = self._get_install_path()
+
+        lib_dir = os.path.join(INSTALL_DIR, "lib")
+
+        # Copy ROOT installation tree to the ROOT package directory in the pip installation path
+        self.copy_tree(INSTALL_DIR, os.path.join(install_path, "ROOT"))
+
+        # Copy cppyy packages separately
+        self.copy_tree(os.path.join(lib_dir, "cppyy"), os.path.join(install_path, "cppyy"))
+        self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), os.path.join(install_path, "cppyy_backend"))
+
+        # Finally copy CPython extension libraries
+        self.copy_file(os.path.join(lib_dir, "libcppyy.so"), install_path)
+        self.copy_file(os.path.join(lib_dir, "libcppyy_backend.so"), install_path)
+        self.copy_file(os.path.join(lib_dir, "ROOT", "libROOTPythonizations.so"), os.path.join(install_path, "ROOT"))
+
+    def get_outputs(self):
+        outputs = _install.get_outputs(self)
+        return outputs
+
+
+class DummyExtension(Extension):
+    """
+    Dummy CPython extension for setuptools setup.
+
+    In order to generate the wheel with CPython extension metadata (i.e.
+    producing one wheel per supported Python version), setuptools requires that
+    at least one CPython extension is declared in the `ext_modules` kwarg passed
+    to the `setup` function. Usually, declaring a CPython extension triggers
+    compilation of the corresponding sources, but in this case we already do
+    that in the CMake build step. This class defines a dummy extension that
+    can be declared to setuptools while avoiding any further compilation step.
+    """
+
+    def __init__(_):
+        super().__init__(name="Dummy", sources=[])
+
+
+pkgs = find_packages("bindings/pyroot/pythonizations/python") + find_packages(
+    "bindings/pyroot/cppyy/cppyy/python", include=["cppyy"]
+)
+
+s = setup(
+    long_description=LONG_DESCRIPTION,
+    package_dir={"": "bindings/pyroot/pythonizations/python", "cppyy": "bindings/pyroot/cppyy/cppyy/python"},
+    packages=pkgs,
+    # Crucial to signal this is not a pure Python package
+    ext_modules=[DummyExtension()],
+    cmdclass={"build": ROOTBuild, "install": ROOTInstall},
+)