From 02e9f6b7879350482b8125636058eedad3f3c45d Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Tue, 9 Jan 2024 17:57:12 +0100 Subject: [PATCH 01/25] pip install ROOT: the first seed --- .../pythonizations/python/ROOT/_facade.py | 5 + .../pythonizations/python/ROOT/_rootcli.py | 13 ++ pyproject.toml | 20 +++ setup.py | 122 ++++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_rootcli.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index 3218ba04ff99d..68f1d3fe48000 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -11,6 +11,8 @@ from ._numbadeclare import _NumbaDeclareDecorator from ._pythonization import pythonization +# Brutal, but necessary for now +from . import _rootcli as _rootexe class PyROOTConfiguration(object): """Class for configuring PyROOT""" @@ -56,6 +58,9 @@ def _subimport(name): class ROOTFacade(types.ModuleType): """Facade class for ROOT module""" + # Brutal, but necessary for now + _rootcli = _rootexe + def __init__(self, module, is_ipython): types.ModuleType.__init__(self, module.__name__) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py b/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py new file mode 100644 index 0000000000000..eab27dde3d761 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py @@ -0,0 +1,13 @@ +import os, sys, subprocess + +MYHOME = os.path.dirname(__file__) + +def main(): + os.environ['LD_LIBRARY_PATH'] = os.path.join(MYHOME, 'lib') + rootexe = os.path.join(MYHOME, 'bin', 'root.exe') + + return subprocess.call(rootexe) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000..1e93b63c9ca93 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["cmake", "setuptools", "wheel"] + +[project] +name = "root-cern" +version = "0.1a3" + +requires-python = ">=3.8" +authors = [ + {name = "Vincenzo Eduardo Padulano", email="vincenzo.eduardo.padulano@cern.ch"} +] +maintainers = [ + {name = "Vincenzo Eduardo Padulano", email = "vincenzo.eduardo.padulano@cern.ch"} +] +description = "An open-source data analysis framework used by high energy physics and others." +readme = "README.md" +license = {file = "LICENSE"} + +[project.scripts] +root = "ROOT._rootcli:main" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000..7a3d0e83c5507 --- /dev/null +++ b/setup.py @@ -0,0 +1,122 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +import pathlib +from setuptools.command.build import build as _build +from setuptools.command.install import install as _install + +import subprocess +import os +import glob +import shlex +here = pathlib.Path(__file__).parent.resolve() + +# Get the long description from the README file +long_description = (here / "README.md").read_text(encoding="utf-8") + +SOURCE_DIR = os.path.expanduser("~/Programs/rootproject/rootsrc") +BUILD_DIR = os.path.expanduser("~/Programs/rootproject/pip-tests/build") +INSTALL_DIR = os.path.expanduser("~/Programs/rootproject/pip-tests/install") + + +class my_cmake_build(_build): + def run(self): + # base run + _build.run(self) + + if not os.path.exists(SOURCE_DIR): + raise RuntimeError("Could not find the source directory!") + + if not os.path.exists(BUILD_DIR): + os.makedirs(BUILD_DIR) + + if not os.path.exists(INSTALL_DIR): + os.makedirs(INSTALL_DIR) + + base_opts = shlex.split("cmake -GNinja -Dccache=ON") + mode_opts = shlex.split("-Dminimal=ON -Dpyroot=ON -Druntime_cxxmodules=ON") + dirs_opts = shlex.split( + f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") + configure_command = base_opts + mode_opts + dirs_opts + subprocess.run(configure_command, check=True) + + build_command = f"cmake --build {BUILD_DIR} -j{os.cpu_count()}" + subprocess.run(shlex.split(build_command), check=True) + + + +class my_install(_install): + def _get_install_path(self): + # depending on goal, copy over pre-installed tree + if hasattr(self, 'bdist_dir') and self.bdist_dir: + install_path = self.bdist_dir + else: + install_path = self.install_lib + + print(f"\n\n{install_path=}\n\n") + print(f"\n\n{self.install_usersite=}\n\n") + print(f"\n\n{self.install_userbase=}\n\n") + + return install_path + + def run(self): + # base install + _install.run(self) + + install_cmd = f"cmake --build {BUILD_DIR} --target install" + subprocess.run(shlex.split(install_cmd), check=True) + + install_path = self._get_install_path() + + lib_dir = os.path.join(INSTALL_DIR, "lib") + install_libdir = os.path.join(install_path, "ROOT", "lib") + + self.copy_tree(os.path.join(INSTALL_DIR, "lib"), os.path.join(install_path, "ROOT", "lib")) + self.copy_tree(os.path.join(INSTALL_DIR, "include"), os.path.join(install_path, "ROOT", "include")) + self.copy_tree(os.path.join(INSTALL_DIR, "bin"), os.path.join(install_path, "ROOT", "bin")) + self.copy_tree(os.path.join(INSTALL_DIR, "etc"), os.path.join(install_path, "ROOT", "etc")) + + self.copy_tree(os.path.join(lib_dir, "cppyy"), + os.path.join(install_path, "cppyy")) + self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), + os.path.join(install_path, "cppyy_backend")) + + extlibs = ["libcppyy3_11.so", "libcppyy_backend3_11.so", "libROOTPythonizations3_11.so"] + for ext in extlibs: + self.copy_file(os.path.join(lib_dir, ext), install_path) + + cppextensions = [os.path.join(install_path, ext) for ext in extlibs] + for ext in cppextensions: + subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN:\$ORIGIN/ROOT/lib {ext}")) + + rootlibs = glob.glob(os.path.join(install_libdir, "lib*.so")) + for lib in rootlibs: + subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN {lib}")) + + def get_outputs(self): + outputs = _install.get_outputs(self) + return outputs + + +pkgs = ( + find_packages('bindings/pyroot/pythonizations/python') + + find_packages('bindings/pyroot/cppyy/cppyy/python', include=['cppyy']) +) + +s = setup( + + package_dir={'': 'bindings/pyroot/pythonizations/python', + 'cppyy': 'bindings/pyroot/cppyy/cppyy/python'}, + packages=pkgs, + + cmdclass={ + 'build': my_cmake_build, + 'install': my_install}, + +) From e6fd58311eb5a686c17e54f686ffa85e106a326a Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Sat, 20 Jan 2024 09:46:41 +0100 Subject: [PATCH 02/25] new setup for cibuildwheel --- pyproject.toml | 7 ++- setup.py | 8 +-- setup_local_pip.py | 122 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 setup_local_pip.py diff --git a/pyproject.toml b/pyproject.toml index 1e93b63c9ca93..55a15344ef43e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["cmake", "setuptools", "wheel"] +requires = ["cmake", "setuptools", "wheel", "ninja"] [project] name = "root-cern" @@ -18,3 +18,8 @@ license = {file = "LICENSE"} [project.scripts] root = "ROOT._rootcli:main" + +[tool.cibuildwheel] +# Increase pip debugging output +build-verbosity = 1 + diff --git a/setup.py b/setup.py index 7a3d0e83c5507..bb25f03bc2780 100644 --- a/setup.py +++ b/setup.py @@ -20,9 +20,9 @@ # Get the long description from the README file long_description = (here / "README.md").read_text(encoding="utf-8") -SOURCE_DIR = os.path.expanduser("~/Programs/rootproject/rootsrc") -BUILD_DIR = os.path.expanduser("~/Programs/rootproject/pip-tests/build") -INSTALL_DIR = os.path.expanduser("~/Programs/rootproject/pip-tests/install") +SOURCE_DIR = os.path.join(os.getcwd(), "rootsrc") +BUILD_DIR = os.path.join(os.getcwd(), "myrootbuild") +INSTALL_DIR = os.path.join(os.getcwd(), "myrootinstall") class my_cmake_build(_build): @@ -31,7 +31,7 @@ def run(self): _build.run(self) if not os.path.exists(SOURCE_DIR): - raise RuntimeError("Could not find the source directory!") + subprocess.run(shlex.split("git clone https://github.com/root-project/root.git rootsrc"), check=True) if not os.path.exists(BUILD_DIR): os.makedirs(BUILD_DIR) diff --git a/setup_local_pip.py b/setup_local_pip.py new file mode 100644 index 0000000000000..59b5459ded1ef --- /dev/null +++ b/setup_local_pip.py @@ -0,0 +1,122 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +import pathlib +from setuptools.command.build import build as _build +from setuptools.command.install import install as _install + +import subprocess +import os +import glob +import shlex +here = pathlib.Path(__file__).parent.resolve() + +# Get the long description from the README file +long_description = (here / "README.md").read_text(encoding="utf-8") + +SOURCE_DIR = os.path.expanduser("~/programs/rootproject/rootsrc") +BUILD_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/build") +INSTALL_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/install") + + +class my_cmake_build(_build): + def run(self): + # base run + _build.run(self) + + if not os.path.exists(SOURCE_DIR): + raise RuntimeError("Could not find the source directory!") + + if not os.path.exists(BUILD_DIR): + os.makedirs(BUILD_DIR) + + if not os.path.exists(INSTALL_DIR): + os.makedirs(INSTALL_DIR) + + base_opts = shlex.split("cmake -GNinja -Dccache=ON") + mode_opts = shlex.split("-Dminimal=ON -Dpyroot=ON -Druntime_cxxmodules=ON") + dirs_opts = shlex.split( + f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") + configure_command = base_opts + mode_opts + dirs_opts + subprocess.run(configure_command, check=True) + + build_command = f"cmake --build {BUILD_DIR} -j{os.cpu_count()}" + subprocess.run(shlex.split(build_command), check=True) + + + +class my_install(_install): + def _get_install_path(self): + # depending on goal, copy over pre-installed tree + if hasattr(self, 'bdist_dir') and self.bdist_dir: + install_path = self.bdist_dir + else: + install_path = self.install_lib + + print(f"\n\n{install_path=}\n\n") + print(f"\n\n{self.install_usersite=}\n\n") + print(f"\n\n{self.install_userbase=}\n\n") + + return install_path + + def run(self): + # base install + _install.run(self) + + install_cmd = f"cmake --build {BUILD_DIR} --target install" + subprocess.run(shlex.split(install_cmd), check=True) + + install_path = self._get_install_path() + + lib_dir = os.path.join(INSTALL_DIR, "lib") + install_libdir = os.path.join(install_path, "ROOT", "lib") + + self.copy_tree(os.path.join(INSTALL_DIR, "lib"), os.path.join(install_path, "ROOT", "lib")) + self.copy_tree(os.path.join(INSTALL_DIR, "include"), os.path.join(install_path, "ROOT", "include")) + self.copy_tree(os.path.join(INSTALL_DIR, "bin"), os.path.join(install_path, "ROOT", "bin")) + self.copy_tree(os.path.join(INSTALL_DIR, "etc"), os.path.join(install_path, "ROOT", "etc")) + + self.copy_tree(os.path.join(lib_dir, "cppyy"), + os.path.join(install_path, "cppyy")) + self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), + os.path.join(install_path, "cppyy_backend")) + + extlibs = ["libcppyy3_11.so", "libcppyy_backend3_11.so", "libROOTPythonizations3_11.so"] + for ext in extlibs: + self.copy_file(os.path.join(lib_dir, ext), install_path) + + cppextensions = [os.path.join(install_path, ext) for ext in extlibs] + for ext in cppextensions: + subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN:\$ORIGIN/ROOT/lib {ext}")) + + rootlibs = glob.glob(os.path.join(install_libdir, "lib*.so")) + for lib in rootlibs: + subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN {lib}")) + + def get_outputs(self): + outputs = _install.get_outputs(self) + return outputs + + +pkgs = ( + find_packages('bindings/pyroot/pythonizations/python') + + find_packages('bindings/pyroot/cppyy/cppyy/python', include=['cppyy']) +) + +s = setup( + + package_dir={'': 'bindings/pyroot/pythonizations/python', + 'cppyy': 'bindings/pyroot/cppyy/cppyy/python'}, + packages=pkgs, + + cmdclass={ + 'build': my_cmake_build, + 'install': my_install}, + +) From 470c709d0829d67a0ed3a574d63dd061de91405c Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Sat, 20 Jan 2024 22:00:01 +0100 Subject: [PATCH 03/25] Automate version of C++ Python extension libs --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bb25f03bc2780..7f964b1ae7568 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ import os import glob import shlex +import sys here = pathlib.Path(__file__).parent.resolve() # Get the long description from the README file @@ -87,7 +88,8 @@ def run(self): self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), os.path.join(install_path, "cppyy_backend")) - extlibs = ["libcppyy3_11.so", "libcppyy_backend3_11.so", "libROOTPythonizations3_11.so"] + libs_version = f"{sys.version_info.major}_{sys.version_info.minor}" + extlibs = [f"libcppyy{libs_version}.so", f"libcppyy_backend{libs_version}.so", f"libROOTPythonizations{libs_version}.so"] for ext in extlibs: self.copy_file(os.path.join(lib_dir, ext), install_path) From 6017070a1585080b3220a3421fe8dff2baa81e86 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 25 Jan 2024 00:00:47 +0100 Subject: [PATCH 04/25] First steps towards building Python wheels Removing the need to link against libPython is a requirement for manylinux. A run of `cibuildwheel` with this configuration produces a wheel, which can be reinstalled on another Python virtual environment in the same machine. The problem is that the wheel is recognized as a pure Python wheel and the C++ libraries are just copied verbatim over. We need to understand how to build according to the Python version. --- bindings/jupyroot/CMakeLists.txt | 21 ++++++++++++++++ bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt | 1 + .../pyroot/cppyy/cppyy-backend/CMakeLists.txt | 1 + bindings/pyroot/pythonizations/CMakeLists.txt | 1 + cmake/modules/SearchInstalledSoftware.cmake | 4 ++-- pyproject.toml | 2 +- setup.py | 24 +++++++++---------- 7 files changed, 39 insertions(+), 15 deletions(-) diff --git a/bindings/jupyroot/CMakeLists.txt b/bindings/jupyroot/CMakeLists.txt index 259f737756d65..0bd7199af58d6 100644 --- a/bindings/jupyroot/CMakeLists.txt +++ b/bindings/jupyroot/CMakeLists.txt @@ -39,6 +39,27 @@ add_custom_command(OUTPUT ${output_files} DEPENDS ${input_files}) add_custom_target(JupyROOT ALL DEPENDS ${output_files}) +<<<<<<< HEAD +======= +set(libname JupyROOT) + +# libJupyROOT uses ROOT headers from source dirs and depends on Core +add_library(${libname} SHARED src/IOHandler.cxx) +# Set the suffix to '.so' and the prefix to 'lib' +set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +if(MSVC) + set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) + set_target_properties(${libname} PROPERTIES SUFFIX ".pyd") + target_link_libraries(${libname} PUBLIC Core) +elseif(APPLE) + target_link_libraries(${libname} PUBLIC -Wl,-bind_at_load -Wl,-w -Wl,-undefined -Wl,dynamic_lookup Core) +else() + target_link_libraries(${libname} PUBLIC -Wl,--unresolved-symbols=ignore-all Core) +endif() + +target_include_directories(${libname} SYSTEM PRIVATE ${PYTHON_INCLUDE_DIRS}) + +>>>>>>> b7e3c8fba6 (First steps towards building Python wheels) # Compile .py files foreach(py_source ${py_sources}) install(CODE "execute_process(COMMAND ${Python3_EXECUTABLE} -m py_compile ${localruntimedir}/${py_source})") diff --git a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt index 911517294bb6a..9fdededca012f 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt +++ b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt @@ -54,6 +54,7 @@ set(libname cppyy) add_library(${libname} SHARED ${headers} ${sources}) # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") if(MSVC) target_link_libraries(${libname} PUBLIC cppyy_backend Python3::Python) set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) diff --git a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt index f296f3886db91..f3fe95e90c549 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt +++ b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt @@ -31,6 +31,7 @@ if(MSVC) endif() # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") target_link_libraries(${libname} Core ${CMAKE_DL_LIBS}) # cppyy uses ROOT headers from binary directory diff --git a/bindings/pyroot/pythonizations/CMakeLists.txt b/bindings/pyroot/pythonizations/CMakeLists.txt index 9a16d7d2887ca..950b1f66cc0e5 100644 --- a/bindings/pyroot/pythonizations/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/CMakeLists.txt @@ -168,6 +168,7 @@ file(COPY ${ROOT_headers_dir}/ DESTINATION ${CMAKE_BINARY_DIR}/include/ROOT) set(libname ROOTPythonizations) add_library(${libname} SHARED ${cpp_sources}) +set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") # Insert the ROOTPythonizationsPySources in the dependency graph add_dependencies(${libname} ROOTPythonizationsPySources) diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 5a7e7b47f50cc..edf01a8bc7e05 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -677,7 +677,7 @@ set(Python3_FIND_FRAMEWORK LAST) # Even if we don't build PyROOT, one still need python executable to run some scripts list(APPEND python_components Interpreter) if(pyroot OR tmva-pymva) - list(APPEND python_components Development) + list(APPEND python_components Development.Module) endif() if(tmva-pymva) list(APPEND python_components NumPy) @@ -1759,7 +1759,7 @@ endif(tmva) #---Check for PyROOT--------------------------------------------------------------------- if(pyroot) - if(Python3_Development_FOUND) + if(Python3_Development.Module_FOUND) message(STATUS "PyROOT: development package found. Building for version ${Python3_VERSION}") else() if(fail-on-missing) diff --git a/pyproject.toml b/pyproject.toml index 55a15344ef43e..8e2a49c0a0008 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["cmake", "setuptools", "wheel", "ninja"] +requires = ["cmake", "setuptools", "wheel", "ninja", "numpy"] [project] name = "root-cern" diff --git a/setup.py b/setup.py index 7f964b1ae7568..d002f4b8a02bc 100644 --- a/setup.py +++ b/setup.py @@ -21,9 +21,9 @@ # Get the long description from the README file long_description = (here / "README.md").read_text(encoding="utf-8") -SOURCE_DIR = os.path.join(os.getcwd(), "rootsrc") -BUILD_DIR = os.path.join(os.getcwd(), "myrootbuild") -INSTALL_DIR = os.path.join(os.getcwd(), "myrootinstall") +SOURCE_DIR = os.getcwd() +BUILD_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/build") +INSTALL_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/install") class my_cmake_build(_build): @@ -31,8 +31,8 @@ def run(self): # base run _build.run(self) - if not os.path.exists(SOURCE_DIR): - subprocess.run(shlex.split("git clone https://github.com/root-project/root.git rootsrc"), check=True) + # if not os.path.exists(SOURCE_DIR): + # subprocess.run(shlex.split("git clone https://github.com/root-project/root.git rootsrc"), check=True) if not os.path.exists(BUILD_DIR): os.makedirs(BUILD_DIR) @@ -41,7 +41,7 @@ def run(self): os.makedirs(INSTALL_DIR) base_opts = shlex.split("cmake -GNinja -Dccache=ON") - mode_opts = shlex.split("-Dminimal=ON -Dpyroot=ON -Druntime_cxxmodules=ON") + mode_opts = shlex.split("-Dminimal=ON -Drpath=ON -Dpyroot=ON -Druntime_cxxmodules=ON") dirs_opts = shlex.split( f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") configure_command = base_opts + mode_opts + dirs_opts @@ -93,13 +93,13 @@ def run(self): for ext in extlibs: self.copy_file(os.path.join(lib_dir, ext), install_path) - cppextensions = [os.path.join(install_path, ext) for ext in extlibs] - for ext in cppextensions: - subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN:\$ORIGIN/ROOT/lib {ext}")) + # cppextensions = [os.path.join(install_path, ext) for ext in extlibs] + # for ext in cppextensions: + # subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN:\$ORIGIN/ROOT/lib {ext}")) - rootlibs = glob.glob(os.path.join(install_libdir, "lib*.so")) - for lib in rootlibs: - subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN {lib}")) + # rootlibs = glob.glob(os.path.join(install_libdir, "lib*.so")) + # for lib in rootlibs: + # subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN {lib}")) def get_outputs(self): outputs = _install.get_outputs(self) From 1366d54af2b374b5d06cbea8b200302d420b52dd Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Wed, 31 Jan 2024 18:52:04 +0100 Subject: [PATCH 05/25] Convince cibuildwheel ROOT is not pure Python Apparently just declaring a C extension module in the setuptools setup function is enough. Even if that extension has nothing to do with the project and doesn't exist anywhere. --- pyproject.toml | 2 +- setup.py | 36 ++++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8e2a49c0a0008..deab3c495e41b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,4 +22,4 @@ root = "ROOT._rootcli:main" [tool.cibuildwheel] # Increase pip debugging output build-verbosity = 1 - +build="cp3{8,9,10,11,12}-manylinux_x86_64" diff --git a/setup.py b/setup.py index d002f4b8a02bc..2498e3a9a07b4 100644 --- a/setup.py +++ b/setup.py @@ -6,14 +6,15 @@ """ # Always prefer setuptools over distutils -from setuptools import setup, find_packages +from setuptools import setup, find_packages, Extension import pathlib +import shutil +import tempfile from setuptools.command.build import build as _build from setuptools.command.install import install as _install import subprocess import os -import glob import shlex import sys here = pathlib.Path(__file__).parent.resolve() @@ -22,8 +23,8 @@ long_description = (here / "README.md").read_text(encoding="utf-8") SOURCE_DIR = os.getcwd() -BUILD_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/build") -INSTALL_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/install") +BUILD_DIR = tempfile.mkdtemp() +INSTALL_DIR = tempfile.mkdtemp() class my_cmake_build(_build): @@ -93,19 +94,22 @@ def run(self): for ext in extlibs: self.copy_file(os.path.join(lib_dir, ext), install_path) - # cppextensions = [os.path.join(install_path, ext) for ext in extlibs] - # for ext in cppextensions: - # subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN:\$ORIGIN/ROOT/lib {ext}")) - - # rootlibs = glob.glob(os.path.join(install_libdir, "lib*.so")) - # for lib in rootlibs: - # subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN {lib}")) - def get_outputs(self): outputs = _install.get_outputs(self) return outputs +class CMakeExtension(Extension): + """ + An extension to run the cmake build + + This simply overrides the base extension class so that setuptools + doesn't try to build your sources for you + """ + + def __init__(self, name, sources=[]): + super().__init__(name = name, sources = sources) + pkgs = ( find_packages('bindings/pyroot/pythonizations/python') + find_packages('bindings/pyroot/cppyy/cppyy/python', include=['cppyy']) @@ -116,9 +120,17 @@ def get_outputs(self): package_dir={'': 'bindings/pyroot/pythonizations/python', 'cppyy': 'bindings/pyroot/cppyy/cppyy/python'}, packages=pkgs, + ext_modules=[ + CMakeExtension(name="pippo"), + ], cmdclass={ 'build': my_cmake_build, 'install': my_install}, ) + + +# Cleanup temporary directories +shutil.rmtree(BUILD_DIR) +shutil.rmtree(INSTALL_DIR) From 8dcd28452390a1dfa70a4aef2c8068d8b33775d2 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Tue, 26 Mar 2024 22:54:51 +0100 Subject: [PATCH 06/25] Enabling RDataFrame and RooFit --- pyproject.toml | 8 +++++++- setup.py | 29 +++++++++++++++++------------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index deab3c495e41b..4fe33a6fd277e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["cmake", "setuptools", "wheel", "ninja", "numpy"] [project] name = "root-cern" -version = "0.1a3" +version = "0.1a4" requires-python = ">=3.8" authors = [ @@ -23,3 +23,9 @@ root = "ROOT._rootcli:main" # Increase pip debugging output build-verbosity = 1 build="cp3{8,9,10,11,12}-manylinux_x86_64" +#build="cp312-manylinux_x86_64" +manylinux-x86_64-image = "manylinux_2_28" + +# Install system library +[tool.cibuildwheel.linux] +before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel tbb tbb-devel xrootd-client xrootd-client-devel" diff --git a/setup.py b/setup.py index 2498e3a9a07b4..adc311513102c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ https://github.com/pypa/sampleproject """ -# Always prefer setuptools over distutils from setuptools import setup, find_packages, Extension import pathlib import shutil @@ -26,6 +25,9 @@ BUILD_DIR = tempfile.mkdtemp() INSTALL_DIR = tempfile.mkdtemp() +# SOURCE_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/rootsrc".split("/")) +# BUILD_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/build".split("/")) +# INSTALL_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/install".split("/")) class my_cmake_build(_build): def run(self): @@ -42,7 +44,10 @@ def run(self): os.makedirs(INSTALL_DIR) base_opts = shlex.split("cmake -GNinja -Dccache=ON") - mode_opts = shlex.split("-Dminimal=ON -Drpath=ON -Dpyroot=ON -Druntime_cxxmodules=ON") + mode_opts = shlex.split( + "-Dgminimal=ON -Dasimage=ON -Drpath=ON -Dpyroot=ON " + "-Druntime_cxxmodules=ON -Ddataframe=ON -Dxrootd=ON -Dimt=ON " + "-Droofit=ON") dirs_opts = shlex.split( f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") configure_command = base_opts + mode_opts + dirs_opts @@ -77,20 +82,20 @@ def run(self): install_path = self._get_install_path() lib_dir = os.path.join(INSTALL_DIR, "lib") - install_libdir = os.path.join(install_path, "ROOT", "lib") - self.copy_tree(os.path.join(INSTALL_DIR, "lib"), os.path.join(install_path, "ROOT", "lib")) - self.copy_tree(os.path.join(INSTALL_DIR, "include"), os.path.join(install_path, "ROOT", "include")) - self.copy_tree(os.path.join(INSTALL_DIR, "bin"), os.path.join(install_path, "ROOT", "bin")) - self.copy_tree(os.path.join(INSTALL_DIR, "etc"), os.path.join(install_path, "ROOT", "etc")) + # Copy ROOT installation tree to the ROOT package directory in the pip installation path + self.copy_tree(INSTALL_DIR, os.path.join(install_path, "ROOT")) + # self.copy_tree(os.path.join(INSTALL_DIR, "lib"), os.path.join(install_path, "ROOT", "lib")) + # self.copy_tree(os.path.join(INSTALL_DIR, "include"), os.path.join(install_path, "ROOT", "include")) + # self.copy_tree(os.path.join(INSTALL_DIR, "bin"), os.path.join(install_path, "ROOT", "bin")) + # self.copy_tree(os.path.join(INSTALL_DIR, "etc"), os.path.join(install_path, "ROOT", "etc")) self.copy_tree(os.path.join(lib_dir, "cppyy"), os.path.join(install_path, "cppyy")) self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), os.path.join(install_path, "cppyy_backend")) - libs_version = f"{sys.version_info.major}_{sys.version_info.minor}" - extlibs = [f"libcppyy{libs_version}.so", f"libcppyy_backend{libs_version}.so", f"libROOTPythonizations{libs_version}.so"] + extlibs = ["libcppyy.so", "libcppyy_backend.so", "libROOTPythonizations.so"] for ext in extlibs: self.copy_file(os.path.join(lib_dir, ext), install_path) @@ -131,6 +136,6 @@ def __init__(self, name, sources=[]): ) -# Cleanup temporary directories -shutil.rmtree(BUILD_DIR) -shutil.rmtree(INSTALL_DIR) +# # Cleanup temporary directories +# shutil.rmtree(BUILD_DIR) +# shutil.rmtree(INSTALL_DIR) From aa2f5c1a5f4d789e6d1ce9b1d1c67559b6fbf3cc Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Wed, 8 May 2024 15:09:11 +0200 Subject: [PATCH 07/25] Rename the project to ROOT --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4fe33a6fd277e..1146e45b4c9a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,8 @@ requires = ["cmake", "setuptools", "wheel", "ninja", "numpy"] [project] -name = "root-cern" -version = "0.1a4" +name = "ROOT" +version = "0.1a5" requires-python = ">=3.8" authors = [ From b5b029e3be234f6d05f4688939a065388b0f44fc Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Fri, 10 May 2024 11:00:50 +0200 Subject: [PATCH 08/25] TPython requires linking against libPython --- bindings/jupyroot/CMakeLists.txt | 21 --------------------- setup.py | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/bindings/jupyroot/CMakeLists.txt b/bindings/jupyroot/CMakeLists.txt index 0bd7199af58d6..259f737756d65 100644 --- a/bindings/jupyroot/CMakeLists.txt +++ b/bindings/jupyroot/CMakeLists.txt @@ -39,27 +39,6 @@ add_custom_command(OUTPUT ${output_files} DEPENDS ${input_files}) add_custom_target(JupyROOT ALL DEPENDS ${output_files}) -<<<<<<< HEAD -======= -set(libname JupyROOT) - -# libJupyROOT uses ROOT headers from source dirs and depends on Core -add_library(${libname} SHARED src/IOHandler.cxx) -# Set the suffix to '.so' and the prefix to 'lib' -set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) -if(MSVC) - set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) - set_target_properties(${libname} PROPERTIES SUFFIX ".pyd") - target_link_libraries(${libname} PUBLIC Core) -elseif(APPLE) - target_link_libraries(${libname} PUBLIC -Wl,-bind_at_load -Wl,-w -Wl,-undefined -Wl,dynamic_lookup Core) -else() - target_link_libraries(${libname} PUBLIC -Wl,--unresolved-symbols=ignore-all Core) -endif() - -target_include_directories(${libname} SYSTEM PRIVATE ${PYTHON_INCLUDE_DIRS}) - ->>>>>>> b7e3c8fba6 (First steps towards building Python wheels) # Compile .py files foreach(py_source ${py_sources}) install(CODE "execute_process(COMMAND ${Python3_EXECUTABLE} -m py_compile ${localruntimedir}/${py_source})") diff --git a/setup.py b/setup.py index adc311513102c..1a11f33b2b3ad 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,11 @@ # Get the long description from the README file long_description = (here / "README.md").read_text(encoding="utf-8") -SOURCE_DIR = os.getcwd() +# SOURCE_DIR = os.getcwd() BUILD_DIR = tempfile.mkdtemp() INSTALL_DIR = tempfile.mkdtemp() -# SOURCE_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/rootsrc".split("/")) +SOURCE_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/rootsrc".split("/")) # BUILD_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/build".split("/")) # INSTALL_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/install".split("/")) From 1160c36f338e9666950c4dd6d7d33d847e0abcd7 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Wed, 28 Aug 2024 10:38:19 +0200 Subject: [PATCH 09/25] setuptools >= 72 breaks cmake extension compatibility See https://github.com/pypa/distutils/issues/284 , the issue is marked as solved but it is not really. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1146e45b4c9a3..416401d978bf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["cmake", "setuptools", "wheel", "ninja", "numpy"] +requires = ["cmake", "setuptools<72", "wheel", "ninja", "numpy"] [project] name = "ROOT" From f57f9e8395a1e84561c8e6d29d4ec140207fa667 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Wed, 28 Aug 2024 14:14:48 +0200 Subject: [PATCH 10/25] use non-specific directories --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1a11f33b2b3ad..adc311513102c 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,11 @@ # Get the long description from the README file long_description = (here / "README.md").read_text(encoding="utf-8") -# SOURCE_DIR = os.getcwd() +SOURCE_DIR = os.getcwd() BUILD_DIR = tempfile.mkdtemp() INSTALL_DIR = tempfile.mkdtemp() -SOURCE_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/rootsrc".split("/")) +# SOURCE_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/rootsrc".split("/")) # BUILD_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/build".split("/")) # INSTALL_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/install".split("/")) From 899fda2c5b69c5f9d14e664c568738a158728803 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Fri, 11 Oct 2024 13:27:23 +0200 Subject: [PATCH 11/25] Cleanup setup.py and check for C++ compiler --- .../pythonizations/python/ROOT/__init__.py | 20 ++- pyproject.toml | 4 +- setup.py | 75 ++++------- setup_local_pip.py | 122 ------------------ 4 files changed, 43 insertions(+), 178 deletions(-) delete mode 100644 setup_local_pip.py diff --git a/bindings/pyroot/pythonizations/python/ROOT/__init__.py b/bindings/pyroot/pythonizations/python/ROOT/__init__.py index 0cdf689ca2166..d12a7058795d6 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/__init__.py +++ b/bindings/pyroot/pythonizations/python/ROOT/__init__.py @@ -8,9 +8,27 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ -import importlib +# First, check for presence of important dependencies import os +import shlex import sys +import subprocess +import textwrap + +try: + # Mimic what cling does to find standard headers include path + cmd = shlex.split('c++ -xc++ -E -v /dev/null') + env = os.environ + env.update({'LC_ALL': 'C'}) + subprocess.run(cmd, env=env, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +except Exception as e: + msg = ( + "Could not find a C++ compiler when importing ROOT. Make sure a C++ compiler as well as the C++ standard " + "libraries are installed. For example, run `[apt,dnf] install g++` or follow similar instructions for your " + "distribution. For more info, visit https://root.cern/install/dependencies" + ) + raise ImportError(textwrap.fill(msg, width=80)) from e # Prevent cppyy's check for the PCH os.environ["CLING_STANDARD_PCH"] = "none" diff --git a/pyproject.toml b/pyproject.toml index 416401d978bf7..82126343bc350 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["cmake", "setuptools<72", "wheel", "ninja", "numpy"] [project] name = "ROOT" -version = "0.1a5" +version = "0.1a6" requires-python = ">=3.8" authors = [ @@ -26,6 +26,6 @@ build="cp3{8,9,10,11,12}-manylinux_x86_64" #build="cp312-manylinux_x86_64" manylinux-x86_64-image = "manylinux_2_28" -# Install system library +# Install system libraries [tool.cibuildwheel.linux] before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel tbb tbb-devel xrootd-client xrootd-client-devel" diff --git a/setup.py b/setup.py index adc311513102c..13fda540952b9 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ from setuptools import setup, find_packages, Extension import pathlib -import shutil import tempfile from setuptools.command.build import build as _build from setuptools.command.install import install as _install @@ -15,65 +14,46 @@ import subprocess import os import shlex -import sys -here = pathlib.Path(__file__).parent.resolve() # Get the long description from the README file -long_description = (here / "README.md").read_text(encoding="utf-8") +SOURCE_DIR = pathlib.Path(__file__).parent.resolve() +LONG_DESCRIPTION = (SOURCE_DIR / "README.md").read_text(encoding="utf-8") -SOURCE_DIR = os.getcwd() BUILD_DIR = tempfile.mkdtemp() INSTALL_DIR = tempfile.mkdtemp() -# SOURCE_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/rootsrc".split("/")) -# BUILD_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/build".split("/")) -# INSTALL_DIR = os.path.join(os.sep, *"host/home/vpadulan/Programs/rootproject/pip-tests/install".split("/")) -class my_cmake_build(_build): +class ROOTBuild(_build): def run(self): - # base run _build.run(self) - # if not os.path.exists(SOURCE_DIR): - # subprocess.run(shlex.split("git clone https://github.com/root-project/root.git rootsrc"), check=True) - - if not os.path.exists(BUILD_DIR): - os.makedirs(BUILD_DIR) - - if not os.path.exists(INSTALL_DIR): - os.makedirs(INSTALL_DIR) - - base_opts = shlex.split("cmake -GNinja -Dccache=ON") + # Configure ROOT build + base_opts = shlex.split("cmake -GNinja") mode_opts = shlex.split( - "-Dgminimal=ON -Dasimage=ON -Drpath=ON -Dpyroot=ON " - "-Druntime_cxxmodules=ON -Ddataframe=ON -Dxrootd=ON -Dimt=ON " + "-Druntime_cxxmodules=ON -Drpath=ON " # Generic build configuration + "-Dgminimal=ON -Dasimage=ON " # Graphics + "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dimt=ON " "-Droofit=ON") dirs_opts = shlex.split( f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") configure_command = base_opts + mode_opts + dirs_opts subprocess.run(configure_command, check=True) + # Run build with CMake build_command = f"cmake --build {BUILD_DIR} -j{os.cpu_count()}" subprocess.run(shlex.split(build_command), check=True) - -class my_install(_install): +class ROOTInstall(_install): def _get_install_path(self): - # depending on goal, copy over pre-installed tree if hasattr(self, 'bdist_dir') and self.bdist_dir: install_path = self.bdist_dir else: install_path = self.install_lib - print(f"\n\n{install_path=}\n\n") - print(f"\n\n{self.install_usersite=}\n\n") - print(f"\n\n{self.install_userbase=}\n\n") - return install_path def run(self): - # base install _install.run(self) install_cmd = f"cmake --build {BUILD_DIR} --target install" @@ -85,17 +65,16 @@ def run(self): # Copy ROOT installation tree to the ROOT package directory in the pip installation path self.copy_tree(INSTALL_DIR, os.path.join(install_path, "ROOT")) - # self.copy_tree(os.path.join(INSTALL_DIR, "lib"), os.path.join(install_path, "ROOT", "lib")) - # self.copy_tree(os.path.join(INSTALL_DIR, "include"), os.path.join(install_path, "ROOT", "include")) - # self.copy_tree(os.path.join(INSTALL_DIR, "bin"), os.path.join(install_path, "ROOT", "bin")) - # self.copy_tree(os.path.join(INSTALL_DIR, "etc"), os.path.join(install_path, "ROOT", "etc")) + # Copy cppyy packages separately self.copy_tree(os.path.join(lib_dir, "cppyy"), os.path.join(install_path, "cppyy")) self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), os.path.join(install_path, "cppyy_backend")) - extlibs = ["libcppyy.so", "libcppyy_backend.so", "libROOTPythonizations.so"] + # Finally copy CPython extension libraries + extlibs = ["libcppyy.so", "libcppyy_backend.so", + "libROOTPythonizations.so"] for ext in extlibs: self.copy_file(os.path.join(lib_dir, ext), install_path) @@ -104,16 +83,14 @@ def get_outputs(self): return outputs -class CMakeExtension(Extension): +class DummyExtension(Extension): """ - An extension to run the cmake build - This simply overrides the base extension class so that setuptools doesn't try to build your sources for you """ + def __init__(_): + super().__init__(name="Dummy", sources=[]) - def __init__(self, name, sources=[]): - super().__init__(name = name, sources = sources) pkgs = ( find_packages('bindings/pyroot/pythonizations/python') + @@ -121,21 +98,13 @@ def __init__(self, name, sources=[]): ) s = setup( - + long_description=LONG_DESCRIPTION, package_dir={'': 'bindings/pyroot/pythonizations/python', 'cppyy': 'bindings/pyroot/cppyy/cppyy/python'}, packages=pkgs, - ext_modules=[ - CMakeExtension(name="pippo"), - ], - + # Crucial to signal this is not a pure Python package + ext_modules=[DummyExtension()], cmdclass={ - 'build': my_cmake_build, - 'install': my_install}, - + 'build': ROOTBuild, + 'install': ROOTInstall}, ) - - -# # Cleanup temporary directories -# shutil.rmtree(BUILD_DIR) -# shutil.rmtree(INSTALL_DIR) diff --git a/setup_local_pip.py b/setup_local_pip.py deleted file mode 100644 index 59b5459ded1ef..0000000000000 --- a/setup_local_pip.py +++ /dev/null @@ -1,122 +0,0 @@ -"""A setuptools based setup module. - -See: -https://packaging.python.org/guides/distributing-packages-using-setuptools/ -https://github.com/pypa/sampleproject -""" - -# Always prefer setuptools over distutils -from setuptools import setup, find_packages -import pathlib -from setuptools.command.build import build as _build -from setuptools.command.install import install as _install - -import subprocess -import os -import glob -import shlex -here = pathlib.Path(__file__).parent.resolve() - -# Get the long description from the README file -long_description = (here / "README.md").read_text(encoding="utf-8") - -SOURCE_DIR = os.path.expanduser("~/programs/rootproject/rootsrc") -BUILD_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/build") -INSTALL_DIR = os.path.expanduser("~/programs/rootproject/pip-tests/install") - - -class my_cmake_build(_build): - def run(self): - # base run - _build.run(self) - - if not os.path.exists(SOURCE_DIR): - raise RuntimeError("Could not find the source directory!") - - if not os.path.exists(BUILD_DIR): - os.makedirs(BUILD_DIR) - - if not os.path.exists(INSTALL_DIR): - os.makedirs(INSTALL_DIR) - - base_opts = shlex.split("cmake -GNinja -Dccache=ON") - mode_opts = shlex.split("-Dminimal=ON -Dpyroot=ON -Druntime_cxxmodules=ON") - dirs_opts = shlex.split( - f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") - configure_command = base_opts + mode_opts + dirs_opts - subprocess.run(configure_command, check=True) - - build_command = f"cmake --build {BUILD_DIR} -j{os.cpu_count()}" - subprocess.run(shlex.split(build_command), check=True) - - - -class my_install(_install): - def _get_install_path(self): - # depending on goal, copy over pre-installed tree - if hasattr(self, 'bdist_dir') and self.bdist_dir: - install_path = self.bdist_dir - else: - install_path = self.install_lib - - print(f"\n\n{install_path=}\n\n") - print(f"\n\n{self.install_usersite=}\n\n") - print(f"\n\n{self.install_userbase=}\n\n") - - return install_path - - def run(self): - # base install - _install.run(self) - - install_cmd = f"cmake --build {BUILD_DIR} --target install" - subprocess.run(shlex.split(install_cmd), check=True) - - install_path = self._get_install_path() - - lib_dir = os.path.join(INSTALL_DIR, "lib") - install_libdir = os.path.join(install_path, "ROOT", "lib") - - self.copy_tree(os.path.join(INSTALL_DIR, "lib"), os.path.join(install_path, "ROOT", "lib")) - self.copy_tree(os.path.join(INSTALL_DIR, "include"), os.path.join(install_path, "ROOT", "include")) - self.copy_tree(os.path.join(INSTALL_DIR, "bin"), os.path.join(install_path, "ROOT", "bin")) - self.copy_tree(os.path.join(INSTALL_DIR, "etc"), os.path.join(install_path, "ROOT", "etc")) - - self.copy_tree(os.path.join(lib_dir, "cppyy"), - os.path.join(install_path, "cppyy")) - self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), - os.path.join(install_path, "cppyy_backend")) - - extlibs = ["libcppyy3_11.so", "libcppyy_backend3_11.so", "libROOTPythonizations3_11.so"] - for ext in extlibs: - self.copy_file(os.path.join(lib_dir, ext), install_path) - - cppextensions = [os.path.join(install_path, ext) for ext in extlibs] - for ext in cppextensions: - subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN:\$ORIGIN/ROOT/lib {ext}")) - - rootlibs = glob.glob(os.path.join(install_libdir, "lib*.so")) - for lib in rootlibs: - subprocess.run(shlex.split(f"patchelf --set-rpath \$ORIGIN {lib}")) - - def get_outputs(self): - outputs = _install.get_outputs(self) - return outputs - - -pkgs = ( - find_packages('bindings/pyroot/pythonizations/python') + - find_packages('bindings/pyroot/cppyy/cppyy/python', include=['cppyy']) -) - -s = setup( - - package_dir={'': 'bindings/pyroot/pythonizations/python', - 'cppyy': 'bindings/pyroot/cppyy/cppyy/python'}, - packages=pkgs, - - cmdclass={ - 'build': my_cmake_build, - 'install': my_install}, - -) From e274dfff575f0c9a21bcc1e7c11b71ee800add35 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Fri, 11 Oct 2024 15:05:00 +0200 Subject: [PATCH 12/25] Preserve cmd line args to ROOT cli --- .../pythonizations/python/ROOT/_rootcli.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py b/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py index eab27dde3d761..25bd1b5e3e33c 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_rootcli.py @@ -1,12 +1,24 @@ -import os, sys, subprocess +import os +import subprocess +import sys -MYHOME = os.path.dirname(__file__) +ROOT_HOME = os.path.dirname(__file__) -def main(): - os.environ['LD_LIBRARY_PATH'] = os.path.join(MYHOME, 'lib') - rootexe = os.path.join(MYHOME, 'bin', 'root.exe') - return subprocess.call(rootexe) +def main() -> None: + # Ensure all ROOT libraries are found at runtime by the process + os.environ['LD_LIBRARY_PATH'] = os.path.join(ROOT_HOME, 'lib') + # Finds the ROOT process from the current installation directory + rootexe = os.path.join(ROOT_HOME, 'bin', 'root.exe') + if not os.path.exists(rootexe): + msg = (f"Could not find 'root' executable in directory '{os.path.join(ROOT_HOME, "bin")}'. " + "Something is wrong in the ROOT installation.") + raise FileNotFoundError(msg) + # Make sure command line arguments are preserved + args = [rootexe] + sys.argv[1:] + # Run the actual ROOT executable and return the exit code to the main Python process + out = subprocess.run(args) + return out.returncode if __name__ == "__main__": From 7ebf749513e04899df5b938c62d96b7722b335e8 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Sun, 13 Oct 2024 16:33:34 +0200 Subject: [PATCH 13/25] Establish full list of build options with fail-on-missing --- pyproject.toml | 2 +- setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82126343bc350..360782f8dcb0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,4 +28,4 @@ manylinux-x86_64-image = "manylinux_2_28" # Install system libraries [tool.cibuildwheel.linux] -before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel tbb tbb-devel xrootd-client xrootd-client-devel" +before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel" diff --git a/setup.py b/setup.py index 13fda540952b9..2fa92c452d872 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,9 @@ def run(self): # Configure ROOT build base_opts = shlex.split("cmake -GNinja") mode_opts = shlex.split( - "-Druntime_cxxmodules=ON -Drpath=ON " # Generic build configuration - "-Dgminimal=ON -Dasimage=ON " # Graphics + "-Dbuiltin_nlohmannjson=ON -Dbuiltin_tbb=ON -Dbuiltin_xrootd=ON -Dbuiltin_lz4=ON -Dbuiltin_lzma=ON -Dbuiltin_zstd=ON -Dbuiltin_xxhash=ON" # builtins + "-Druntime_cxxmodules=ON -Drpath=ON -Dfail-on-missing=ON " # Generic build configuration + "-Dgminimal=ON -Dasimage=ON -Dopengl=OFF " # Graphics "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dimt=ON " "-Droofit=ON") dirs_opts = shlex.split( From e222c2608c270d3d757bf00b346e60b7762a0ff0 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Sun, 13 Oct 2024 16:56:01 +0200 Subject: [PATCH 14/25] Document some of the choices --- bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt | 11 ++++++ .../pyroot/cppyy/cppyy-backend/CMakeLists.txt | 11 ++++++ bindings/pyroot/pythonizations/CMakeLists.txt | 11 ++++++ .../pythonizations/python/ROOT/_facade.py | 7 +++- cmake/modules/SearchInstalledSoftware.cmake | 2 + setup.py | 38 +++++++++++++++---- 6 files changed, 70 insertions(+), 10 deletions(-) diff --git a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt index 9fdededca012f..47c537c12566c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt +++ b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt @@ -54,6 +54,17 @@ set(libname cppyy) add_library(${libname} SHARED ${headers} ${sources}) # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- libROOTPythonizations.so +# -- ROOT/ +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") if(MSVC) target_link_libraries(${libname} PUBLIC cppyy_backend Python3::Python) diff --git a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt index f3fe95e90c549..9760862b7cfef 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt +++ b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt @@ -31,6 +31,17 @@ if(MSVC) endif() # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES}) +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- libROOTPythonizations.so +# -- ROOT/ +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") target_link_libraries(${libname} Core ${CMAKE_DL_LIBS}) diff --git a/bindings/pyroot/pythonizations/CMakeLists.txt b/bindings/pyroot/pythonizations/CMakeLists.txt index 950b1f66cc0e5..641e585fc3d55 100644 --- a/bindings/pyroot/pythonizations/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/CMakeLists.txt @@ -168,6 +168,17 @@ file(COPY ${ROOT_headers_dir}/ DESTINATION ${CMAKE_BINARY_DIR}/include/ROOT) set(libname ROOTPythonizations) add_library(${libname} SHARED ${cpp_sources}) +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- libROOTPythonizations.so +# -- ROOT/ +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") # Insert the ROOTPythonizationsPySources in the dependency graph diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index 68f1d3fe48000..43fd05af219ec 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -11,7 +11,11 @@ from ._numbadeclare import _NumbaDeclareDecorator from ._pythonization import pythonization -# Brutal, but necessary for now +# Because the facade acts as the actual ROOT package for `import` purposes, +# we have to expose any further scripts that must be picked up by the pip +# build backend so that the generated executable can properly run +# `from ROOT._name_of_script import main`. See pyproject.toml for the list +# of enabled scripts. from . import _rootcli as _rootexe class PyROOTConfiguration(object): @@ -58,7 +62,6 @@ def _subimport(name): class ROOTFacade(types.ModuleType): """Facade class for ROOT module""" - # Brutal, but necessary for now _rootcli = _rootexe def __init__(self, module, is_ipython): diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index edf01a8bc7e05..fe1d093221577 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -677,6 +677,8 @@ set(Python3_FIND_FRAMEWORK LAST) # Even if we don't build PyROOT, one still need python executable to run some scripts list(APPEND python_components Interpreter) if(pyroot OR tmva-pymva) + # We have to only look for the Python development module in order to be able to build ROOT with a pip backend + # In particular, it is forbidden to link against libPython.so, see https://peps.python.org/pep-0513/#libpythonx-y-so-1 list(APPEND python_components Development.Module) endif() if(tmva-pymva) diff --git a/setup.py b/setup.py index 2fa92c452d872..671b5a6f059b8 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,22 @@ -"""A setuptools based setup module. - -See: -https://packaging.python.org/guides/distributing-packages-using-setuptools/ -https://github.com/pypa/sampleproject +""" +setuptools-based build of ROOT. + +This script uses setuptools API to steer a custom CMake build of ROOT. All build +configuration options are specified in the class responsible for building. The +installation step produces the following output target install directory: + +``` +site-packages/ +-- cppyy/ +-- cppyy_backend/ +-- libcppyy.so +-- libcppyy_backend.so +-- libROOTPythonizations.so +-- ROOT/ +``` + +A custom extension module is injected in the setuptools setup to properly +generate the wheel with CPython extension metadata. """ from setuptools import setup, find_packages, Extension @@ -30,7 +44,8 @@ def run(self): # Configure ROOT build base_opts = shlex.split("cmake -GNinja") mode_opts = shlex.split( - "-Dbuiltin_nlohmannjson=ON -Dbuiltin_tbb=ON -Dbuiltin_xrootd=ON -Dbuiltin_lz4=ON -Dbuiltin_lzma=ON -Dbuiltin_zstd=ON -Dbuiltin_xxhash=ON" # builtins + "-Dbuiltin_nlohmannjson=ON -Dbuiltin_tbb=ON -Dbuiltin_xrootd=ON " # builtins + "-Dbuiltin_lz4=ON -Dbuiltin_lzma=ON -Dbuiltin_zstd=ON -Dbuiltin_xxhash=ON" # builtins "-Druntime_cxxmodules=ON -Drpath=ON -Dfail-on-missing=ON " # Generic build configuration "-Dgminimal=ON -Dasimage=ON -Dopengl=OFF " # Graphics "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dimt=ON " @@ -86,8 +101,15 @@ def get_outputs(self): class DummyExtension(Extension): """ - This simply overrides the base extension class so that setuptools - doesn't try to build your sources for you + Dummy CPython extension for setuptools setup. + + In order to generate the wheel with CPython extension metadata (i.e. + producing one wheel per supported Python version), setuptools requires that + at least one CPython extension is declared in the `ext_modules` kwarg passed + to the `setup` function. Usually, declaring a CPython extension triggers + compilation of the corresponding sources, but in this case we already do + that in the CMake build step. This class defines a dummy extension that + can be declared to setuptools while avoiding any further compilation step. """ def __init__(_): super().__init__(name="Dummy", sources=[]) From 459a18b820b3c321dd04396af70b7d7ce076ba01 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Sun, 13 Oct 2024 16:57:25 +0200 Subject: [PATCH 15/25] Update list of Python versions for cibuildwheel --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 360782f8dcb0a..e2d5dc5b5c4ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,7 @@ root = "ROOT._rootcli:main" [tool.cibuildwheel] # Increase pip debugging output build-verbosity = 1 -build="cp3{8,9,10,11,12}-manylinux_x86_64" -#build="cp312-manylinux_x86_64" +build="cp3{8,9,10,11,12,13}-manylinux_x86_64" manylinux-x86_64-image = "manylinux_2_28" # Install system libraries From 4717b70da18885c4b2674dab1a8a5c4724b53092 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 11:11:14 +0200 Subject: [PATCH 16/25] Rebase to master and add github workflow --- .github/workflows/python_wheel_build.yml | 54 +++++++++++++++++++ bindings/pyroot/pythonizations/CMakeLists.txt | 29 +++++----- pyproject.toml | 7 +-- setup.py | 43 +++++++-------- 4 files changed, 92 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/python_wheel_build.yml diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml new file mode 100644 index 0000000000000..8d26fda0fba47 --- /dev/null +++ b/.github/workflows/python_wheel_build.yml @@ -0,0 +1,54 @@ +name: 'ROOT Python wheels' + +on: + workflow_dispatch: + inputs: + branch: + description: 'The branch for which the Python wheel has to be generated' + type: string + required: true + default: "experimental-pip-install-root" + +jobs: + python_wheel: + runs-on: ubuntu-latest + env: + WORKDIR: /home/runner/work/root + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: Install Kerberos utilities + run: sudo apt-get install -y krb5-user + + - name: Install XRootD client + run: sudo apt-get -y install xrootd-client + + - name: Build wheels + uses: pypa/cibuildwheel@v3.0.1 + + # Upload artifact to github + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-test + path: ./wheelhouse/*.whl + + - name: Upload wheel to EOS + env: + RWEBEOS_KT: ${{ secrets.RWEBEOS_KT }} + KT_FILE_NAME: /tmp/decoded.keytab + EOS_PATH: /eos/project/r/root-eos/www/experimental-python-wheels + EOS_ENDPOINT: root://eosproject-r.cern.ch + KRB5CCNAME: /tmp/krb5cc + working-directory: ${{ env.WORKDIR }} + run: | + echo +++ Content + ls + echo +++ Retrieving the secret + echo ${RWEBEOS_KT} | base64 -d > ${KT_FILE_NAME} + echo +++ Creating the token + kinit -p ${{ secrets.KRB5USER }}@${{ secrets.KRB5REALM }} -kt ${KT_FILE_NAME} + echo +++ Running the copy + xrdcp -rf wheelhouse ${EOS_ENDPOINT}/${EOS_PATH}/ diff --git a/bindings/pyroot/pythonizations/CMakeLists.txt b/bindings/pyroot/pythonizations/CMakeLists.txt index 641e585fc3d55..5f49ed2ceaeb1 100644 --- a/bindings/pyroot/pythonizations/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/CMakeLists.txt @@ -168,25 +168,13 @@ file(COPY ${ROOT_headers_dir}/ DESTINATION ${CMAKE_BINARY_DIR}/include/ROOT) set(libname ROOTPythonizations) add_library(${libname} SHARED ${cpp_sources}) -# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. -# After the `pip install` step, the target install user directory will look like: -# site-packages/ -# -- cppyy/ -# -- cppyy_backend/ -# -- libcppyy.so -# -- libcppyy_backend.so -# -- libROOTPythonizations.so -# -- ROOT/ -# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension -# shared libraries. -set_target_properties(${libname} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:$ORIGIN/ROOT/lib") # Insert the ROOTPythonizationsPySources in the dependency graph add_dependencies(${libname} ROOTPythonizationsPySources) # Set the suffix to '.so' and the prefix to 'lib' set_target_properties(${libname} PROPERTIES ${ROOT_LIBRARY_PROPERTIES_NO_VERSION}) -target_link_libraries(${libname} PUBLIC Core Tree cppyy) +target_link_libraries(${libname} PUBLIC ROOT::Core ROOT::Tree cppyy) if(MSVC) set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) set_target_properties(${libname} PROPERTIES SUFFIX ".pyd") @@ -240,6 +228,18 @@ else() LIBRARY_OUTPUT_DIRECTORY ${pymoduledir_build}) endif() +# It is crucial to add the path to the other ROOT libraries for compatibility with pip build backends. +# After the `pip install` step, the target install user directory will look like: +# site-packages/ +# -- cppyy/ +# -- cppyy_backend/ +# -- libcppyy.so +# -- libcppyy_backend.so +# -- ROOT/ +# -- -- libROOTPythonizations.so +# With most of the ROOT libraries being stored in `ROOT/lib`. The RPATH must then be updated for the CPython extension +# shared libraries. + if(NOT MSVC) # Make sure that relative RUNPATH to main ROOT libraries is always correct. @@ -254,7 +254,8 @@ if(NOT MSVC) else() set_target_properties(${libname} PROPERTIES BUILD_RPATH "$ORIGIN/${pymoduledir_to_libdir_build}" - INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}" + # ATTENTION: NEXT PART IS CRUCIAL SEE COMMENT ABOVE + INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}:${CMAKE_INSTALL_RPATH}:$ORIGIN/lib" ) endif() diff --git a/pyproject.toml b/pyproject.toml index e2d5dc5b5c4ce..6e62f1331c0ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["cmake", "setuptools<72", "wheel", "ninja", "numpy"] [project] name = "ROOT" -version = "0.1a6" +version = "0.1a7" requires-python = ">=3.8" authors = [ @@ -22,9 +22,10 @@ root = "ROOT._rootcli:main" [tool.cibuildwheel] # Increase pip debugging output build-verbosity = 1 -build="cp3{8,9,10,11,12,13}-manylinux_x86_64" +#build="cp3{8,9,10,11,12,13}-manylinux_x86_64" +build="cp313-manylinux_x86_64" manylinux-x86_64-image = "manylinux_2_28" # Install system libraries [tool.cibuildwheel.linux] -before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel" +before-all = "dnf install -y epel-release && /usr/bin/crb enable && dnf install -y openssl-devel libX11-devel libXpm-devel libXft-devel libXext-devel libuuid-devel libjpeg-devel giflib-devel" diff --git a/setup.py b/setup.py index 671b5a6f059b8..33b66144d20b9 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,8 @@ -- cppyy_backend/ -- libcppyy.so -- libcppyy_backend.so --- libROOTPythonizations.so -- ROOT/ +-- -- libROOTPythonizations.so ``` A custom extension module is injected in the setuptools setup to properly @@ -42,17 +42,18 @@ def run(self): _build.run(self) # Configure ROOT build - base_opts = shlex.split("cmake -GNinja") + base_opts = shlex.split("cmake -GNinja -Dccache=ON") mode_opts = shlex.split( "-Dbuiltin_nlohmannjson=ON -Dbuiltin_tbb=ON -Dbuiltin_xrootd=ON " # builtins "-Dbuiltin_lz4=ON -Dbuiltin_lzma=ON -Dbuiltin_zstd=ON -Dbuiltin_xxhash=ON" # builtins "-Druntime_cxxmodules=ON -Drpath=ON -Dfail-on-missing=ON " # Generic build configuration "-Dgminimal=ON -Dasimage=ON -Dopengl=OFF " # Graphics - "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dimt=ON " - "-Droofit=ON") - dirs_opts = shlex.split( - f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") + "-Dpyroot=ON -Ddataframe=ON -Dxrootd=ON -Dssl=ON -Dimt=ON " + "-Droofit=ON" + ) + dirs_opts = shlex.split(f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR} -B {BUILD_DIR} -S {SOURCE_DIR}") configure_command = base_opts + mode_opts + dirs_opts + print(f"\n\n{' '.join(configure_command)}\n\n") subprocess.run(configure_command, check=True) # Run build with CMake @@ -62,7 +63,7 @@ def run(self): class ROOTInstall(_install): def _get_install_path(self): - if hasattr(self, 'bdist_dir') and self.bdist_dir: + if hasattr(self, "bdist_dir") and self.bdist_dir: install_path = self.bdist_dir else: install_path = self.install_lib @@ -83,16 +84,13 @@ def run(self): self.copy_tree(INSTALL_DIR, os.path.join(install_path, "ROOT")) # Copy cppyy packages separately - self.copy_tree(os.path.join(lib_dir, "cppyy"), - os.path.join(install_path, "cppyy")) - self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), - os.path.join(install_path, "cppyy_backend")) + self.copy_tree(os.path.join(lib_dir, "cppyy"), os.path.join(install_path, "cppyy")) + self.copy_tree(os.path.join(lib_dir, "cppyy_backend"), os.path.join(install_path, "cppyy_backend")) # Finally copy CPython extension libraries - extlibs = ["libcppyy.so", "libcppyy_backend.so", - "libROOTPythonizations.so"] - for ext in extlibs: - self.copy_file(os.path.join(lib_dir, ext), install_path) + self.copy_file(os.path.join(lib_dir, "libcppyy.so"), install_path) + self.copy_file(os.path.join(lib_dir, "libcppyy_backend.so"), install_path) + self.copy_file(os.path.join(lib_dir, "ROOT", "libROOTPythonizations.so"), os.path.join(install_path, "ROOT")) def get_outputs(self): outputs = _install.get_outputs(self) @@ -103,7 +101,7 @@ class DummyExtension(Extension): """ Dummy CPython extension for setuptools setup. - In order to generate the wheel with CPython extension metadata (i.e. + In order to generate the wheel with CPython extension metadata (i.e. producing one wheel per supported Python version), setuptools requires that at least one CPython extension is declared in the `ext_modules` kwarg passed to the `setup` function. Usually, declaring a CPython extension triggers @@ -111,23 +109,20 @@ class DummyExtension(Extension): that in the CMake build step. This class defines a dummy extension that can be declared to setuptools while avoiding any further compilation step. """ + def __init__(_): super().__init__(name="Dummy", sources=[]) -pkgs = ( - find_packages('bindings/pyroot/pythonizations/python') + - find_packages('bindings/pyroot/cppyy/cppyy/python', include=['cppyy']) +pkgs = find_packages("bindings/pyroot/pythonizations/python") + find_packages( + "bindings/pyroot/cppyy/cppyy/python", include=["cppyy"] ) s = setup( long_description=LONG_DESCRIPTION, - package_dir={'': 'bindings/pyroot/pythonizations/python', - 'cppyy': 'bindings/pyroot/cppyy/cppyy/python'}, + package_dir={"": "bindings/pyroot/pythonizations/python", "cppyy": "bindings/pyroot/cppyy/cppyy/python"}, packages=pkgs, # Crucial to signal this is not a pure Python package ext_modules=[DummyExtension()], - cmdclass={ - 'build': ROOTBuild, - 'install': ROOTInstall}, + cmdclass={"build": ROOTBuild, "install": ROOTInstall}, ) From 4000c1222abeb87ca4921f8bc11a0c9d074cd969 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 11:47:20 +0200 Subject: [PATCH 17/25] Run workflow on pull_request --- .github/workflows/python_wheel_build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 8d26fda0fba47..5aab3b6c2bb6f 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -8,6 +8,9 @@ on: type: string required: true default: "experimental-pip-install-root" + pull_request: + branches: + - '**' jobs: python_wheel: From f2e88aceaf0f6b4bdbcf089c129a65cf516ecffc Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 13:41:13 +0200 Subject: [PATCH 18/25] Tweak directory location in workflow --- .github/workflows/python_wheel_build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 5aab3b6c2bb6f..5fd9a104c174c 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -31,12 +31,15 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v3.0.1 + with: + output-dir: ${{ env.WORKDIR }}/wheelhouse # Upload artifact to github - uses: actions/upload-artifact@v4 + working-directory: ${{ env.WORKDIR }} with: name: cibw-wheels-test - path: ./wheelhouse/*.whl + path: wheelhouse/*.whl - name: Upload wheel to EOS env: From 5ae65bb2bc8b1e994b922832b53a2d50efb8cba0 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 13:58:35 +0200 Subject: [PATCH 19/25] Another test for the workflow --- .github/workflows/python_wheel_build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 5fd9a104c174c..65503e703446f 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -9,8 +9,6 @@ on: required: true default: "experimental-pip-install-root" pull_request: - branches: - - '**' jobs: python_wheel: From b654d42ee594e094fcfa4dee92d2d63355ab4412 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 14:01:20 +0200 Subject: [PATCH 20/25] Another change in workflow --- .github/workflows/python_wheel_build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 65503e703446f..cc970b577c8cd 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -31,9 +31,9 @@ jobs: uses: pypa/cibuildwheel@v3.0.1 with: output-dir: ${{ env.WORKDIR }}/wheelhouse - - # Upload artifact to github - - uses: actions/upload-artifact@v4 + + - name: Upload wheel to github + uses: actions/upload-artifact@v4 working-directory: ${{ env.WORKDIR }} with: name: cibw-wheels-test From a2da886b30280e890a89987255fe8196513650c2 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 14:03:25 +0200 Subject: [PATCH 21/25] Try with push --- .github/workflows/python_wheel_build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index cc970b577c8cd..55875a8818c91 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -9,6 +9,9 @@ on: required: true default: "experimental-pip-install-root" pull_request: + branches: "master" + push: + branches: "experimental-pip-install-root" jobs: python_wheel: From 0482c079730638e110844d946c48652da4637d56 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 14:11:43 +0200 Subject: [PATCH 22/25] Remove quotes --- .github/workflows/python_wheel_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 55875a8818c91..5c5088dcd2add 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -9,9 +9,9 @@ on: required: true default: "experimental-pip-install-root" pull_request: - branches: "master" + branches: master push: - branches: "experimental-pip-install-root" + branches: experimental-pip-install-root jobs: python_wheel: From cdc83183e629e0ee2c6174a7887e9c08f4562cae Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 14:20:14 +0200 Subject: [PATCH 23/25] Try extending types of pull_request --- .github/workflows/python_wheel_build.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 5c5088dcd2add..2adc4188b1174 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -9,9 +9,12 @@ on: required: true default: "experimental-pip-install-root" pull_request: - branches: master - push: - branches: experimental-pip-install-root + branches: + - master + types: + - opened + - synchronize + - reopened jobs: python_wheel: From 586dd43ef0124a91e3ea6292867b2b9ce6140b5e Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 14:24:53 +0200 Subject: [PATCH 24/25] Just run workflow on push --- .github/workflows/python_wheel_build.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 2adc4188b1174..878ea2159938f 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -8,13 +8,8 @@ on: type: string required: true default: "experimental-pip-install-root" - pull_request: - branches: - - master - types: - - opened - - synchronize - - reopened + push: + branches: experimental-pip-install-root jobs: python_wheel: @@ -39,8 +34,8 @@ jobs: output-dir: ${{ env.WORKDIR }}/wheelhouse - name: Upload wheel to github - uses: actions/upload-artifact@v4 working-directory: ${{ env.WORKDIR }} + uses: actions/upload-artifact@v4 with: name: cibw-wheels-test path: wheelhouse/*.whl From 6b82aa240b8ed62dc521db0b4a3c7f4738801216 Mon Sep 17 00:00:00 2001 From: Vincenzo Eduardo Padulano Date: Thu, 17 Jul 2025 14:26:14 +0200 Subject: [PATCH 25/25] Correct artifact directory --- .github/workflows/python_wheel_build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python_wheel_build.yml b/.github/workflows/python_wheel_build.yml index 878ea2159938f..f9aa68fd9b647 100644 --- a/.github/workflows/python_wheel_build.yml +++ b/.github/workflows/python_wheel_build.yml @@ -34,11 +34,10 @@ jobs: output-dir: ${{ env.WORKDIR }}/wheelhouse - name: Upload wheel to github - working-directory: ${{ env.WORKDIR }} uses: actions/upload-artifact@v4 with: name: cibw-wheels-test - path: wheelhouse/*.whl + path: ${{ env.WORKDIR }}/wheelhouse/*.whl - name: Upload wheel to EOS env: