diff --git a/graalpython/com.oracle.graal.python.frozen/freeze_modules.py b/graalpython/com.oracle.graal.python.frozen/freeze_modules.py index 937236432b..62415caaad 100644 --- a/graalpython/com.oracle.graal.python.frozen/freeze_modules.py +++ b/graalpython/com.oracle.graal.python.frozen/freeze_modules.py @@ -123,7 +123,6 @@ def add_graalpython_core(): "_sre", "_sysconfig", "java", - "pip_hook", ]: modname = f"graalpy.{os.path.basename(name)}" modpath = os.path.join(lib_graalpython, f"{name}.py") diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py b/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py index 21b91b57f1..dc3647c4aa 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py @@ -45,6 +45,7 @@ import tempfile import threading import unittest +import zipfile from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from urllib.parse import urljoin @@ -189,6 +190,34 @@ def run_venv_pip_install(self, package, extra_env=None, assert_stderr_matches=No f"Didn't match expected stderr.\nExpected (regex): {assert_stderr_matches}\nActual:{proc.stderr}" return re.findall(r'Successfully installed (\S+)', proc.stdout) + def run_venv_pip_runner_install(self, package): + runner = subprocess.check_output([ + self.venv_python, + '-c', + 'import pathlib, pip; print(pathlib.Path(pip.__file__).with_name("__pip-runner__.py"))', + ], universal_newlines=True).strip() + proc = subprocess.run( + [ + str(self.venv_python), + runner, + '--isolated', + 'install', + '--force-reinstall', + '--find-links', str(self.index_dir), + '--no-index', + '--no-cache-dir', + package, + ], + check=True, + capture_output=True, + env=self.pip_env, + universal_newlines=True, + ) + print(proc.stdout) + print(proc.stderr) + assert 'Applying GraalPy patch failed for' not in proc.stderr + return re.findall(r'Successfully installed (\S+)', proc.stdout) + def run_test_fun(self): code = "import patched_package; print(patched_package.test_fun())" return subprocess.check_output([self.venv_python, '-c', code], universal_newlines=True).strip() @@ -233,6 +262,35 @@ def test_sdist_patched_version(self): self.run_venv_pip_install('foo') assert self.run_test_fun() == "Patched" + def test_sdist_patched_version_with_pip_runner(self): + self.add_package_to_index('foo', '1.1.0', 'sdist') + self.prepare_config('foo', [{ + 'patch': 'foo-1.1.0.patch', + 'version': '== 1.1.0', + 'subdir': 'src', + }]) + self.run_venv_pip_runner_install('foo') + assert self.run_test_fun() == "Patched" + + def test_mark_wheel_preserves_executable_scripts(self): + import graalpy_pip_extensions + + wheel = self.build_dir / 'executable_script-1.0-py3-none-any.whl' + script = 'executable_script-1.0.data/scripts/executable-script' + with zipfile.ZipFile(wheel, 'w') as z: + info = zipfile.ZipInfo(script) + info.external_attr = 0o100755 << 16 + z.writestr(info, '#!/usr/bin/env python\n') + z.writestr('executable_script-1.0.dist-info/METADATA', 'Name: executable_script\nVersion: 1.0\n') + z.writestr('executable_script-1.0.dist-info/WHEEL', 'Wheel-Version: 1.0\n') + z.writestr('executable_script-1.0.dist-info/RECORD', '') + + graalpy_pip_extensions.mark_wheel(wheel) + + with zipfile.ZipFile(wheel) as z: + mode = z.getinfo(script).external_attr >> 16 + assert mode & 0o111 + def test_different_patch_wheel_sdist1(self): self.add_package_to_index('foo', '1.1.0', 'sdist') self.prepare_config('foo', [ diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_pip_audit_hook.py b/graalpython/com.oracle.graal.python.test/src/tests/test_pip_audit_hook.py new file mode 100644 index 0000000000..8a18a4dc3f --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_pip_audit_hook.py @@ -0,0 +1,526 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + + +def write_fake_pip(root, patched=False): + pip_package = root / "pip" + pip_package.mkdir() + pip_package.joinpath("__init__.py").write_text( + "__GRAALPY_PATCHED = True\n" if patched else "", encoding="utf-8" + ) + return root + + +class PipAuditHookTests(unittest.TestCase): + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + @unittest.skipUnless(shutil.which("patch"), "requires the patch utility") + def test_source_ready_event_applies_local_graalpy_patch_after_unpatched_pip_import(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-") as tmp: + root = Path(tmp) + patches = root / "patches" + source = root / "demo-1.0.0" + fake_pip_root = root / "fake-pip" + patches.mkdir() + source.mkdir() + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + + (patches / "metadata.toml").write_text( + "\n".join( + [ + "[[demo.rules]]", + "version = '== 1.0.0'", + "patch = 'demo.patch'", + "license = 'MIT'", + ] + ), + encoding="utf-8", + ) + (patches / "demo.patch").write_text( + "\n".join( + [ + "diff --git a/demo_module.py b/demo_module.py", + "--- a/demo_module.py", + "+++ b/demo_module.py", + "@@ -1 +1 @@", + '-value = "old"', + '+value = "patched"', + "", + ] + ), + encoding="utf-8", + ) + module = source / "demo_module.py" + module.write_text('value = "old"\n', encoding="utf-8") + + env = os.environ.copy() + env["PIP_GRAALPY_PATCHES_URL"] = patches.as_uri() + env.pop("PIP_GRAALPY_DISABLE_PATCHING", None) + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + ( + "import sys; " + "sys.path.insert(0, sys.argv[1]); " + "import pip; " + "sys.audit('pip.requirement.source_ready', " + "'demo', None, 'https://example.invalid/demo-1.0.0.tar.gz', " + "sys.argv[2], False, 0)" + ), + str(fake_pip_root), + str(source), + ], + env=env, + ) + + self.assertEqual( + module.read_text(encoding="utf-8"), 'value = "patched"\n' + ) + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + @unittest.skipUnless(shutil.which("patch"), "requires the patch utility") + def test_wheel_install_event_rewrites_wheel_before_install(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-wheel-") as tmp: + root = Path(tmp) + patches = root / "patches" + fake_pip_root = root / "fake-pip" + patches.mkdir() + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + wheel_path = root / "demo-1.0.0-py3-none-any.whl" + + (patches / "metadata.toml").write_text( + "\n".join( + [ + "[[demo.rules]]", + "version = '== 1.0.0'", + "dist-type = 'wheel'", + "patch = 'demo.patch'", + "license = 'MIT'", + ] + ), + encoding="utf-8", + ) + (patches / "demo.patch").write_text( + "\n".join( + [ + "diff --git a/demo_module.py b/demo_module.py", + "--- a/demo_module.py", + "+++ b/demo_module.py", + "@@ -1 +1 @@", + '-value = "old"', + '+value = "patched"', + "", + ] + ), + encoding="utf-8", + ) + + with zipfile.ZipFile(wheel_path, "w") as z: + z.writestr("demo_module.py", 'value = "old"\n') + z.writestr( + "demo-1.0.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + z.writestr( + "demo-1.0.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: demo\nVersion: 1.0.0\n", + ) + z.writestr("demo-1.0.0.dist-info/RECORD", "") + + env = os.environ.copy() + env["PIP_GRAALPY_PATCHES_URL"] = patches.as_uri() + env.pop("PIP_GRAALPY_DISABLE_PATCHING", None) + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + ( + "import sys; " + "sys.path.insert(0, sys.argv[1]); " + "import pip; " + "sys.audit('pip.wheel.install', 'demo', sys.argv[2])" + ), + str(fake_pip_root), + str(wheel_path), + ], + env=env, + ) + + with zipfile.ZipFile(wheel_path) as z: + self.assertEqual( + z.read("demo_module.py").decode("utf-8"), 'value = "patched"\n' + ) + self.assertIn("demo-1.0.0.dist-info/GRAALPY_MARKER", z.namelist()) + record = z.read("demo-1.0.0.dist-info/RECORD").decode("utf-8") + self.assertIn("demo_module.py,sha256=", record) + self.assertIn("demo-1.0.0.dist-info/GRAALPY_MARKER,sha256=", record) + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + @unittest.skipUnless(shutil.which("patch"), "requires the patch utility") + def test_patched_pip_does_not_install_audit_hook(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-patched-") as tmp: + root = Path(tmp) + patches = root / "patches" + source = root / "demo-1.0.0" + fake_pip_root = root / "fake-pip" + patches.mkdir() + source.mkdir() + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root, patched=True) + + (patches / "metadata.toml").write_text( + "\n".join( + [ + "[[demo.rules]]", + "version = '== 1.0.0'", + "patch = 'demo.patch'", + "license = 'MIT'", + ] + ), + encoding="utf-8", + ) + (patches / "demo.patch").write_text( + "\n".join( + [ + "diff --git a/demo_module.py b/demo_module.py", + "--- a/demo_module.py", + "+++ b/demo_module.py", + "@@ -1 +1 @@", + '-value = "old"', + '+value = "patched"', + "", + ] + ), + encoding="utf-8", + ) + module = source / "demo_module.py" + module.write_text('value = "old"\n', encoding="utf-8") + + env = os.environ.copy() + env["PIP_GRAALPY_PATCHES_URL"] = patches.as_uri() + env.pop("PIP_GRAALPY_DISABLE_PATCHING", None) + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + ( + "import sys; " + "sys.path.insert(0, sys.argv[1]); " + "import pip; " + "sys.audit('pip.requirement.source_ready', " + "'demo', None, 'https://example.invalid/demo-1.0.0.tar.gz', " + "sys.argv[2], False, 0)" + ), + str(fake_pip_root), + str(source), + ], + env=env, + ) + + self.assertEqual(module.read_text(encoding="utf-8"), 'value = "old"\n') + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + def test_cli_parse_args_event_adds_graalpy_defaults(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-cli-") as tmp: + root = Path(tmp) + fake_pip_root = root / "fake-pip" + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + + env = os.environ.copy() + env.pop("PIP_INDEX_URL", None) + env.pop("PIP_CACHE_DIR", None) + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + ( + "import sys; " + "from types import SimpleNamespace; " + "sys.path.insert(0, sys.argv[1]); " + "import pip; " + "options = SimpleNamespace(extra_index_urls=[], cache_dir=sys.argv[2]); " + "sys.audit('pip.cli.parse_args', options, [], 0); " + "assert options.extra_index_urls == ['https://www.graalvm.org/python/wheels/']; " + "assert options.cache_dir == sys.argv[2] + '-graalpy'" + ), + str(fake_pip_root), + str(root / "pip-cache"), + ], + env=env, + ) + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + def test_import_function_installs_pip_audit_hook(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-import-") as tmp: + root = Path(tmp) + fake_pip_root = root / "fake-pip" + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + + env = os.environ.copy() + env.pop("PIP_INDEX_URL", None) + env.pop("PIP_CACHE_DIR", None) + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + ( + "import sys; " + "from types import SimpleNamespace; " + "sys.path.insert(0, sys.argv[1]); " + "__import__('pip'); " + "options = SimpleNamespace(extra_index_urls=[], cache_dir=sys.argv[2]); " + "sys.audit('pip.cli.parse_args', options, [], 0); " + "assert options.extra_index_urls == ['https://www.graalvm.org/python/wheels/']; " + "assert options.cache_dir == sys.argv[2] + '-graalpy'" + ), + str(fake_pip_root), + str(root / "pip-cache"), + ], + env=env, + ) + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + def test_find_all_candidates_event_adds_metadata_sources(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-candidates-") as tmp: + root = Path(tmp) + patches = root / "patches" + fake_pip_root = root / "fake-pip" + patches.mkdir() + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + + (patches / "metadata.toml").write_text( + "\n".join( + [ + "[[demo.add-sources]]", + "version = '1.0.0'", + "url = 'https://example.invalid/demo/archive/refs/tags/v1.0.0.tar.gz'", + ] + ), + encoding="utf-8", + ) + + env = os.environ.copy() + env["PIP_GRAALPY_PATCHES_URL"] = patches.as_uri() + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + ( + "import sys; " + "sys.path.insert(0, sys.argv[1]); " + "import pip; " + "extra = []; " + "sys.audit('pip.package_finder.find_all_candidates', 'demo', (), extra); " + "assert extra == [('demo', '1.0.0', {" + "'url': 'https://example.invalid/demo/archive/refs/tags/v1.0.0.tar.gz', " + "'filename': 'demo-1.0.0.tar.gz', " + "'comes_from': 'GraalPy compatibility patches', " + "'requires_python': None, " + "'yanked_reason': None})]" + ), + str(fake_pip_root), + ], + env=env, + ) + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + @unittest.skipUnless(shutil.which("patch"), "requires the patch utility") + def test_source_ready_event_uses_link_filename_for_add_source_urls(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-link-filename-") as tmp: + root = Path(tmp) + patches = root / "patches" + source = root / "demo-1.0.0" + fake_pip_root = root / "fake-pip" + patches.mkdir() + source.mkdir() + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + + (patches / "metadata.toml").write_text( + "\n".join( + [ + "[[demo.rules]]", + "version = '== 1.0.0'", + "patch = 'demo.patch'", + "license = 'MIT'", + ] + ), + encoding="utf-8", + ) + (patches / "demo.patch").write_text( + "\n".join( + [ + "diff --git a/demo_module.py b/demo_module.py", + "--- a/demo_module.py", + "+++ b/demo_module.py", + "@@ -1 +1 @@", + '-value = "old"', + '+value = "patched"', + "", + ] + ), + encoding="utf-8", + ) + module = source / "demo_module.py" + module.write_text('value = "old"\n', encoding="utf-8") + + env = os.environ.copy() + env["PIP_GRAALPY_PATCHES_URL"] = patches.as_uri() + env.pop("PIP_GRAALPY_DISABLE_PATCHING", None) + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + "\n".join( + [ + "import sys", + "sys.path.insert(0, sys.argv[1])", + "import pip", + "class Link:", + " filename = 'demo-1.0.0.tar.gz'", + " def __str__(self):", + " return 'https://example.invalid/demo/archive/refs/tags/v1.0.0.tar.gz'", + "sys.audit(", + " 'pip.requirement.source_ready',", + " 'demo',", + " None,", + " Link(),", + " sys.argv[2],", + " False,", + " 0,", + ")", + ] + ), + str(fake_pip_root), + str(source), + ], + env=env, + ) + + self.assertEqual(module.read_text(encoding="utf-8"), 'value = "patched"\n') + + @unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific test") + def test_source_ready_event_warns_about_suggested_versions(self): + with tempfile.TemporaryDirectory(prefix="graalpy-pip-audit-suggest-") as tmp: + root = Path(tmp) + patches = root / "patches" + source = root / "demo-2.0.0" + fake_pip_root = root / "fake-pip" + patches.mkdir() + source.mkdir() + fake_pip_root.mkdir() + write_fake_pip(fake_pip_root) + + (patches / "metadata.toml").write_text( + "\n".join( + [ + "[[demo.rules]]", + "version = '== 1.0.0'", + "patch = 'demo.patch'", + "license = 'MIT'", + ] + ), + encoding="utf-8", + ) + (patches / "demo.patch").write_text("", encoding="utf-8") + + env = os.environ.copy() + env["PIP_GRAALPY_PATCHES_URL"] = patches.as_uri() + + subprocess.check_call( + [ + sys.executable, + "-S", + "-c", + "\n".join( + [ + "import sys, warnings", + "sys.path.insert(0, sys.argv[1])", + "import pip", + "with warnings.catch_warnings(record=True) as caught:", + " warnings.simplefilter('always')", + " sys.audit(", + " 'pip.requirement.source_ready',", + " 'demo',", + " None,", + " 'https://example.invalid/demo-2.0.0.tar.gz',", + " sys.argv[2],", + " False,", + " 0,", + " )", + " assert any('version(s): == 1.0.0' in str(w.message) for w in caught)", + ] + ), + str(fake_pip_root), + str(source), + ], + env=env, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py b/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py index 5480ed3518..c98e7057fc 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py @@ -64,7 +64,6 @@ '_sre', '_sysconfig', 'java', - 'pip_hook', ] + WINDOWS_CORE_MODULES expected_full_startup_modules = expected_nosite_startup_modules + [ diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java index a7793600ef..17b5fec576 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java @@ -442,8 +442,7 @@ private static TruffleString[] getCoreFiles() { T___GRAALPYTHON__, T__SRE, T__SYSCONFIG, - T_JAVA, - toTruffleStringUncached("pip_hook") + T_JAVA }; } @@ -1330,7 +1329,7 @@ private Source getInternalSource(TruffleString basename, TruffleString prefix) { throw e; } - private void loadFile(TruffleString s, TruffleString prefix) { + public void loadFile(TruffleString s, TruffleString prefix) { loadFile(s, s, prefix); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index c17d58aa35..559fea13d7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -1550,4 +1550,16 @@ static PTuple doCreate(long arrowArrayAddr, long arrowSchemaAddr, return PFactory.createTuple(ctx.getLanguage(inliningTarget), new Object[]{arrowSchemaCapsule, arrowArrayCapsule}); } } + + @Builtin(name = "load_file", minNumOfPositionalArgs = 1) + @GenerateNodeFactory + public abstract static class LoadFile extends PythonUnaryBuiltinNode { + @TruffleBoundary + @Specialization + static PNone doit(TruffleString name, + @Bind PythonContext context) { + context.getCore().loadFile(name, context.getCoreHomeOrFail()); + return PNone.NONE; + } + } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java index 235a182347..e0aebd29fa 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java @@ -102,7 +102,6 @@ private static final class Map { private static final PythonFrozenModule GRAALPY__SRE = new PythonFrozenModule("GRAALPY__SRE", null, false); private static final PythonFrozenModule GRAALPY__SYSCONFIG = new PythonFrozenModule("GRAALPY__SYSCONFIG", null, false); private static final PythonFrozenModule GRAALPY_JAVA = new PythonFrozenModule("GRAALPY_JAVA", null, false); - private static final PythonFrozenModule GRAALPY_PIP_HOOK = new PythonFrozenModule("GRAALPY_PIP_HOOK", null, false); } public static final PythonFrozenModule lookup(String name) { @@ -233,8 +232,6 @@ public static final PythonFrozenModule lookup(String name) { return Map.GRAALPY__SYSCONFIG; case "graalpy.java": return Map.GRAALPY_JAVA; - case "graalpy.pip_hook": - return Map.GRAALPY_PIP_HOOK; default: return null; } diff --git a/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py b/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py new file mode 100644 index 0000000000..e963eefdf0 --- /dev/null +++ b/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py @@ -0,0 +1,455 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import abc +import logging +import os +import re +import sys +import tempfile +import zipfile +from types import SimpleNamespace +from contextlib import contextmanager +from pathlib import Path +import tomllib as tomli +from tomllib import TOMLDecodeError +from urllib.parse import urlparse, urljoin, urlunparse +from urllib.request import url2pathname + +try: + from pip._internal.models.link import Link as _PipLink +except ImportError: + _PipLink = object + +MARKER_FILE_NAME = 'GRAALPY_MARKER' +METADATA_FILENAME = 'metadata.toml' +DEFAULT_PATCHES_PATH = Path(__graalpython__.core_home) / 'patches' +VERSION_PARAMETER = '' +DEFAULT_PATCHES_URL = f'https://raw.githubusercontent.com/oracle/graalpython/refs/heads/github/patches/{VERSION_PARAMETER}/graalpython/lib-graalpython/patches/' + +PATCHES_URL = os.environ.get('PIP_GRAALPY_PATCHES_URL', DEFAULT_PATCHES_URL) +DISABLED_PATCHES_URL = 'disabled' +DISABLE_PATCHING = os.environ.get('PIP_GRAALPY_DISABLE_PATCHING', '').lower() in ('true', '1') +DISABLE_VERSION_SELECTION = os.environ.get('PIP_GRAALPY_DISABLE_VERSION_SELECTION', '').lower() in ('true', '1') + +GRAALPY_VERSION = os.environ.get('TEST_PIP_GRAALPY_VERSION', __graalpython__.get_graalvm_version()) + +logger = logging.getLogger(__name__) + + +def canonicalize_name(name): + return re.sub(r"[-_.]+", "-", name).lower() + + +def specifier_contains(specifier, version): + try: + from pip._vendor.packaging.specifiers import SpecifierSet + except ImportError: + try: + from packaging.specifiers import SpecifierSet + except ImportError: + return any( + spec.strip().removeprefix("==").strip() == version + for spec in specifier.split(",") + if spec.strip().startswith("==") + ) + return SpecifierSet(specifier).contains(version) + + +def url_for_file(patches_url, filename): + scheme, netloc, path, params, query, fragment = urlparse(patches_url) + path = urljoin(path, filename) + return urlunparse((scheme, netloc, path, params, query, fragment)) + + +class RepositoryException(Exception): + pass + + +class RepositoryNotFound(RepositoryException): + pass + + +class AbstractPatchRepository(metaclass=abc.ABCMeta): + def __init__(self, metadata: dict): + self._repository = metadata + + @staticmethod + def metadata_from_string(metadata_content) -> dict: + try: + parsed_metadata = tomli.loads(metadata_content) + return {canonicalize_name(name): data for name, data in parsed_metadata.items()} + except TOMLDecodeError as e: + raise RepositoryException(f"'{METADATA_FILENAME}' cannot be parsed: {e}") + + def get_rules(self, name): + if metadata := self._repository.get(canonicalize_name(name)): + return metadata.get('rules') + + def get_add_sources(self, name): + if metadata := self._repository.get(canonicalize_name(name)): + return metadata.get('add-sources') + + def get_priority_for_version(self, name, version): + if rules := self.get_rules(name): + for rule in rules: + if self.rule_matches_version(rule, version): + return rule.get('install-priority', 1) + return 0 + + @staticmethod + def rule_matches_version(rule, version): + return not rule.get('version') or specifier_contains(rule['version'], version) + + def get_suggested_version_specs(self, name): + versions = set() + if rules := self.get_rules(name): + for rule in rules: + if 'patch' in rule and rule.get('install-priority', 1) > 0 and (version := rule.get('version')): + versions.add(version) + return versions + + def get_matching_rule(self, name, requested_version, dist_type): + if metadata := self.get_rules(name): + for rule in metadata: + if rule.get('dist-type', dist_type) != dist_type: + continue + if not self.rule_matches_version(rule, requested_version): + continue + return rule + + @abc.abstractmethod + def resolve_patch(self, patch_name: str): + pass + + +class EmptyRepository(AbstractPatchRepository): + def __init__(self): + super().__init__({}) + + def resolve_patch(self, patch_name: str): + raise AssertionError("Invalid call") + + +class LocalPatchRepository(AbstractPatchRepository): + def __init__(self, patches_path: Path, repository_data: dict): + super().__init__(repository_data) + self.patches_path = patches_path + + @classmethod + def from_path(cls, patches_path: Path): + try: + with open(patches_path / METADATA_FILENAME) as f: + metadata_content = f.read() + except FileNotFoundError: + raise RepositoryNotFound(f"'{METADATA_FILENAME}' does not exist") + except OSError as e: + raise RepositoryException(f"'{METADATA_FILENAME}' cannot be read: {e}") + return cls(patches_path, cls.metadata_from_string(metadata_content)) + + @contextmanager + def resolve_patch(self, patch_name: str): + yield self.patches_path / patch_name + + +class RemotePatchRepository(AbstractPatchRepository): + def __init__(self, patches_url: str, repository_data: dict): + super().__init__(repository_data) + self.patches_url = patches_url + + @staticmethod + def get_session(): + try: + from pip._internal.cli.index_command import _GRAALPY_SESSION + if _GRAALPY_SESSION: + return _GRAALPY_SESSION + except ImportError: + pass + from pip._vendor import requests + return requests.Session() + + @classmethod + def from_url(cls, patches_url: str): + try: + url = url_for_file(patches_url, METADATA_FILENAME) + response = cls.get_session().get(url) + if response.status_code == 404: + raise RepositoryNotFound(f"'{METADATA_FILENAME}' not found") + response.raise_for_status() + if not response.encoding: + response.encoding = 'utf-8' + metadata_content = response.text + except Exception as e: + raise RepositoryException(f"'{METADATA_FILENAME}' cannot be retrieved': {e}") + return cls(patches_url, cls.metadata_from_string(metadata_content)) + + @contextmanager + def resolve_patch(self, patch_name: str): + from pip._vendor import requests + + try: + response = self.get_session().get(url_for_file(self.patches_url, patch_name)) + response.raise_for_status() + except requests.RequestException as e: + logger.warning("Failed to download GraalPy patch '%s': %s", patch_name, e) + yield None + else: + with tempfile.TemporaryDirectory() as tempdir: + if not response.encoding: + response.encoding = 'utf-8' + patch_file = Path(tempdir) / patch_name + with open(patch_file, 'w') as f: + f.write(response.text) + yield patch_file + + +__PATCH_REPOSITORY = None + + +def repository_from_url_or_path(url_or_path): + if '://' not in url_or_path: + return LocalPatchRepository.from_path(Path(url_or_path)) + elif url_or_path.startswith('file:'): + patches_path = Path(url2pathname(urlparse(url_or_path).path)) + return LocalPatchRepository.from_path(patches_path) + else: + patches_url = url_or_path + if not patches_url.endswith('/'): + patches_url += '/' + return RemotePatchRepository.from_url(patches_url) + + +def create_patch_repository(patches_url): + if patches_url and VERSION_PARAMETER in patches_url: + if not GRAALPY_VERSION.endswith('-dev'): + patches_url = patches_url.replace(VERSION_PARAMETER, GRAALPY_VERSION) + else: + logger.debug("Skipping versioned GraalPy patch repository on snapshot build") + patches_url = None + if patches_url and patches_url != DISABLED_PATCHES_URL: + logger.info( + "Loading GraalPy post-release patch repository from %s. " + "This can be controlled with PIP_GRAALPY_PATCHES_URL environment variable. Set to '%s' to disable", + patches_url, DISABLED_PATCHES_URL) + try: + return repository_from_url_or_path(patches_url) + except RepositoryException as e: + if patches_url == DEFAULT_PATCHES_URL and isinstance(e, RepositoryNotFound): + logger.info("No post-release patch repository published yet") + else: + logger.warning("Failed to load GraalPy patch repository: %s", e) + logger.warning("Falling back to bundled GraalPy patch repository") + try: + return LocalPatchRepository.from_path(DEFAULT_PATCHES_PATH) + except RepositoryException as e: + logger.warning("Failed to load internal GraalPy patch repository: %s", e) + return EmptyRepository() + + +def get_patch_repository(): + global __PATCH_REPOSITORY + if not __PATCH_REPOSITORY: + __PATCH_REPOSITORY = create_patch_repository(PATCHES_URL) + return __PATCH_REPOSITORY + + +def apply_graalpy_patches(filename, location, warn_suggested_versions=False): + """ + Applies any GraalPy patches to package extracted from 'filename' into 'location'. + Note that 'location' must be the parent directory of the package directory itself. + For example: /path/to/site-package and not /path/to/site-packages/mypackage. + """ + if DISABLE_PATCHING: + return + + # we expect filename to be something like "pytest-5.4.2-py3-none-any.whl" + archive_name = os.path.basename(filename) + name_ver_match = re.match(r"^(?P.*?)-(?P[^-]+).*?\.(?Ptar\.gz|tar|whl|zip)$", + archive_name, re.I) + if not name_ver_match: + logger.warning(f"GraalPy warning: could not parse package name, version, or format from {archive_name!r}.\n" + "Could not determine if any GraalPy specific patches need to be applied.") + return + + name = name_ver_match.group('name') + version = name_ver_match.group('version') + suffix = name_ver_match.group('suffix') + is_wheel = suffix == "whl" + + if is_wheel and is_wheel_marked(filename): + # We already processed it when building from source + return + + import autopatch_capi + import subprocess + + autopatch_capi.auto_patch_tree(location) + + logger.info(f"Looking for GraalPy patches for {name}") + repository = get_patch_repository() + + if is_wheel: + # patches intended for binary distribution: + rule = repository.get_matching_rule(name, version, 'wheel') + else: + # patches intended for source distribution if applicable + rule = repository.get_matching_rule(name, version, 'sdist') + if not rule: + rule = repository.get_matching_rule(name, version, 'wheel') + if rule and (subdir := rule.get('subdir')): + # we may need to change wd if we are actually patching a source distribution + # with a patch intended for a binary distribution, because in the source + # distribution the actual deployed sources may be in a subdirectory (typically "src") + location = os.path.join(location, subdir) + if rule: + if patch := rule.get('patch'): + with repository.resolve_patch(patch) as patch_path: + if not patch_path: + return + logger.info(f"Patching package {name} using {patch}") + exe = '.exe' if os.name == 'nt' else '' + try: + subprocess.run([f"patch{exe}", "-f", "-d", str(location), "-p1", "-i", str(patch_path)], check=True) + except FileNotFoundError: + logger.warning( + "WARNING: GraalPy needs the 'patch' utility to apply compatibility patches. Please install it using your system's package manager.") + except subprocess.CalledProcessError: + logger.warning(f"Applying GraalPy patch failed for {name}. The package may still work.") + except Exception: + logger.exception(f"Failed to execute patch utility") + elif version_specs := repository.get_suggested_version_specs(name): + logger.info("We have patches to make this package work on GraalVM for some version(s).") + logger.info("If installing or running fails, consider using one of the versions that we have patches for:") + for version_spec in version_specs: + logger.info(f'{name} {version_spec}') + if warn_suggested_versions: + from warnings import warn + warn( + "GraalPy has compatibility patches for " + f"{name} version(s): {', '.join(sorted(version_specs))}. " + "If installing or running this package fails, consider using one of those versions." + ) + + +def apply_graalpy_sort_order(sort_key_func): + if DISABLE_VERSION_SELECTION: + return sort_key_func + + def wrapper(self, candidate): + default_sort_key = sort_key_func(self, candidate) + priority = get_patch_repository().get_priority_for_version(candidate.name, str(candidate.version)) + return priority, default_sort_key + + return wrapper + + +class AddedSourceLink(_PipLink): + def __init__(self, url: str, filename: str): + if _PipLink is object: + self._url = url + else: + super().__init__(url) + self._filename = filename + + @property + def filename(self) -> str: + return self._filename + + @property + def url(self) -> str: + if _PipLink is object: + return self._url + return super().url + + def __str__(self): + return self.url + + +def installation_candidate(name, version, link): + try: + from pip._internal.models.candidate import InstallationCandidate + return InstallationCandidate(name=name, version=version, link=link) + except ImportError: + return SimpleNamespace(name=name, version=version, link=link) + + +def link_for_url(url): + try: + from pip._internal.models.link import Link + return Link(url) + except ImportError: + return AddedSourceLink(url, os.path.basename(urlparse(url).path)) + + +def get_graalpy_candidates(name): + repository = get_patch_repository() + candidates = [] + for add_source in repository.get_add_sources(name) or []: + version = add_source['version'] + url = add_source['url'] + match = re.search(r'\.(tar\.(?:gz|bz2|xz)|zip|whl)$', urlparse(url).path) + assert match, "Couldn't determine URL suffix" + suffix = match.group(1) + # We need to force the filename to match the usual convention, otherwise we won't find a patch + link = AddedSourceLink(url, f'{name}-{version}.{suffix}') + candidates.append(installation_candidate(name=name, version=version, link=link)) + if name == 'graalpy-virtualenv-seeder': + link = link_for_url(Path(os.path.join(sys.base_prefix, 'graalpy_virtualenv_seeder')).resolve().as_uri()) + candidates.append(installation_candidate(name=name, version='0.0.1', link=link)) + return candidates + + +def mark_wheel(path): + if DISABLE_PATCHING: + return + with zipfile.ZipFile(path, 'a') as z: + dist_info = None + for name in z.namelist(): + if m := re.match(r'([^/]+.dist-info)/', name): + dist_info = m.group(1) + break + assert dist_info, "Cannot find .dist_info in built wheel" + marker = f'{dist_info}/{MARKER_FILE_NAME}' + with z.open(marker, 'w'): + pass + + +def is_wheel_marked(path): + with zipfile.ZipFile(path) as z: + return any(re.match(rf'[^/]+.dist-info/{MARKER_FILE_NAME}$', f) for f in z.namelist()) diff --git a/graalpython/lib-graalpython/patches/pip-24.3.1.patch b/graalpython/lib-graalpython/patches/pip-24.3.1.patch index 1fbe76cf08..abfc736862 100644 --- a/graalpython/lib-graalpython/patches/pip-24.3.1.patch +++ b/graalpython/lib-graalpython/patches/pip-24.3.1.patch @@ -66,7 +66,7 @@ index 0d65ce3..63aa513 100644 from pip._internal.utils.misc import build_netloc from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS -+from pip._internal.utils.graalpy import apply_graalpy_sort_order, get_graalpy_candidates ++from graalpy_pip_extensions import apply_graalpy_sort_order, get_graalpy_candidates if TYPE_CHECKING: from pip._vendor.typing_extensions import TypeGuard @@ -99,7 +99,7 @@ index 5c3bce3..f3057d0 100644 import os from typing import Iterable, Optional, Tuple -+from pip._internal.utils.graalpy import AddedSourceLink ++from graalpy_pip_extensions import AddedSourceLink from pip._vendor.requests.models import Response from pip._internal.cli.progress_bars import get_download_progress_renderer @@ -120,7 +120,7 @@ index aef42aa..0dcc357 100644 file.save() record_installed(file.src_record_path, file.dest_path, file.changed) -+ from pip._internal.utils.graalpy import apply_graalpy_patches ++ from graalpy_pip_extensions import apply_graalpy_patches + apply_graalpy_patches(wheel_path, lib_dir) + def pyc_source_file_paths() -> Generator[str, None, None]: @@ -134,7 +134,7 @@ index 6617644..ad52082 100644 from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.direct_url_helpers import direct_url_from_link from pip._internal.utils.misc import normalize_version_info -+from pip._internal.utils import graalpy ++import graalpy_pip_extensions as graalpy from .base import Candidate, Requirement, format_name @@ -147,366 +147,6 @@ index 6617644..ad52082 100644 def get_install_requirement(self) -> Optional[InstallRequirement]: return self._ireq -diff --git a/pip/_internal/utils/graalpy.py b/pip/_internal/utils/graalpy.py -new file mode 100644 -index 0000000..cdc4852 ---- /dev/null -+++ b/pip/_internal/utils/graalpy.py -@@ -0,0 +1,354 @@ -+import abc -+import logging -+import os -+import re -+import sys -+import tempfile -+import zipfile -+from contextlib import contextmanager -+from pathlib import Path -+from tomllib import TOMLDecodeError -+from urllib.parse import urlparse, urljoin, urlunparse -+ -+from pip._internal.models.candidate import InstallationCandidate -+from pip._internal.models.link import Link -+from pip._internal.utils.urls import url_to_path, path_to_url -+from pip._vendor import tomli, requests -+from pip._vendor.packaging.specifiers import SpecifierSet -+from pip._vendor.packaging.utils import canonicalize_name -+from pip._vendor.packaging.version import VERSION_PATTERN -+ -+MARKER_FILE_NAME = 'GRAALPY_MARKER' -+METADATA_FILENAME = 'metadata.toml' -+DEFAULT_PATCHES_PATH = Path(__graalpython__.core_home) / 'patches' -+VERSION_PARAMETER = '' -+DEFAULT_PATCHES_URL = f'https://raw.githubusercontent.com/oracle/graalpython/refs/heads/github/patches/{VERSION_PARAMETER}/graalpython/lib-graalpython/patches/' -+ -+PATCHES_URL = os.environ.get('PIP_GRAALPY_PATCHES_URL', DEFAULT_PATCHES_URL) -+DISABLED_PATCHES_URL = 'disabled' -+DISABLE_PATCHING = os.environ.get('PIP_GRAALPY_DISABLE_PATCHING', '').lower() in ('true', '1') -+DISABLE_VERSION_SELECTION = os.environ.get('PIP_GRAALPY_DISABLE_VERSION_SELECTION', '').lower() in ('true', '1') -+ -+GRAALPY_VERSION = os.environ.get('TEST_PIP_GRAALPY_VERSION', __graalpython__.get_graalvm_version()) -+ -+logger = logging.getLogger(__name__) -+ -+ -+def url_for_file(patches_url, filename): -+ scheme, netloc, path, params, query, fragment = urlparse(patches_url) -+ path = urljoin(path, filename) -+ return urlunparse((scheme, netloc, path, params, query, fragment)) -+ -+ -+class RepositoryException(Exception): -+ pass -+ -+ -+class RepositoryNotFound(RepositoryException): -+ pass -+ -+ -+class AbstractPatchRepository(metaclass=abc.ABCMeta): -+ def __init__(self, metadata: dict): -+ self._repository = metadata -+ -+ @staticmethod -+ def metadata_from_string(metadata_content) -> dict: -+ try: -+ parsed_metadata = tomli.loads(metadata_content) -+ return {canonicalize_name(name): data for name, data in parsed_metadata.items()} -+ except TOMLDecodeError as e: -+ raise RepositoryException(f"'{METADATA_FILENAME}' cannot be parsed: {e}") -+ -+ def get_rules(self, name): -+ if metadata := self._repository.get(canonicalize_name(name)): -+ return metadata.get('rules') -+ -+ def get_add_sources(self, name): -+ if metadata := self._repository.get(canonicalize_name(name)): -+ return metadata.get('add-sources') -+ -+ def get_priority_for_version(self, name, version): -+ if rules := self.get_rules(name): -+ for rule in rules: -+ if self.rule_matches_version(rule, version): -+ return rule.get('install-priority', 1) -+ return 0 -+ -+ @staticmethod -+ def rule_matches_version(rule, version): -+ return not rule.get('version') or SpecifierSet(rule['version']).contains(version) -+ -+ def get_suggested_version_specs(self, name): -+ versions = set() -+ if rules := self.get_rules(name): -+ for rule in rules: -+ if 'patch' in rule and rule.get('install-priority', 1) > 0 and (version := rule.get('version')): -+ versions.add(version) -+ return versions -+ -+ def get_matching_rule(self, name, requested_version, dist_type): -+ if metadata := self.get_rules(name): -+ for rule in metadata: -+ if rule.get('dist-type', dist_type) != dist_type: -+ continue -+ if not self.rule_matches_version(rule, requested_version): -+ continue -+ return rule -+ -+ @abc.abstractmethod -+ def resolve_patch(self, patch_name: str): -+ pass -+ -+ -+class EmptyRepository(AbstractPatchRepository): -+ def __init__(self): -+ super().__init__({}) -+ -+ def resolve_patch(self, patch_name: str): -+ raise AssertionError("Invalid call") -+ -+ -+class LocalPatchRepository(AbstractPatchRepository): -+ def __init__(self, patches_path: Path, repository_data: dict): -+ super().__init__(repository_data) -+ self.patches_path = patches_path -+ -+ @classmethod -+ def from_path(cls, patches_path: Path): -+ try: -+ with open(patches_path / METADATA_FILENAME) as f: -+ metadata_content = f.read() -+ except FileNotFoundError: -+ raise RepositoryNotFound(f"'{METADATA_FILENAME}' does not exist") -+ except OSError as e: -+ raise RepositoryException(f"'{METADATA_FILENAME}' cannot be read: {e}") -+ return cls(patches_path, cls.metadata_from_string(metadata_content)) -+ -+ @contextmanager -+ def resolve_patch(self, patch_name: str): -+ yield self.patches_path / patch_name -+ -+ -+class RemotePatchRepository(AbstractPatchRepository): -+ def __init__(self, patches_url: str, repository_data: dict): -+ super().__init__(repository_data) -+ self.patches_url = patches_url -+ -+ @staticmethod -+ def get_session(): -+ from pip._internal.cli.index_command import _GRAALPY_SESSION -+ return _GRAALPY_SESSION or requests.Session() -+ -+ @classmethod -+ def from_url(cls, patches_url: str): -+ try: -+ url = url_for_file(patches_url, METADATA_FILENAME) -+ response = cls.get_session().get(url) -+ if response.status_code == 404: -+ raise RepositoryNotFound(f"'{METADATA_FILENAME}' not found") -+ response.raise_for_status() -+ if not response.encoding: -+ response.encoding = 'utf-8' -+ metadata_content = response.text -+ except Exception as e: -+ raise RepositoryException(f"'{METADATA_FILENAME}' cannot be retrieved': {e}") -+ return cls(patches_url, cls.metadata_from_string(metadata_content)) -+ -+ @contextmanager -+ def resolve_patch(self, patch_name: str): -+ try: -+ response = self.get_session().get(url_for_file(self.patches_url, patch_name)) -+ response.raise_for_status() -+ except requests.RequestException as e: -+ logger.warning("Failed to download GraalPy patch '%s': %s", patch_name, e) -+ yield None -+ else: -+ with tempfile.TemporaryDirectory() as tempdir: -+ if not response.encoding: -+ response.encoding = 'utf-8' -+ patch_file = Path(tempdir) / patch_name -+ with open(patch_file, 'w') as f: -+ f.write(response.text) -+ yield patch_file -+ -+ -+__PATCH_REPOSITORY = None -+ -+ -+def repository_from_url_or_path(url_or_path): -+ if '://' not in url_or_path: -+ return LocalPatchRepository.from_path(Path(url_or_path)) -+ elif url_or_path.startswith('file:'): -+ patches_path = Path(url_to_path(url_or_path)) -+ return LocalPatchRepository.from_path(patches_path) -+ else: -+ patches_url = url_or_path -+ if not patches_url.endswith('/'): -+ patches_url += '/' -+ return RemotePatchRepository.from_url(patches_url) -+ -+ -+def create_patch_repository(patches_url): -+ if patches_url and VERSION_PARAMETER in patches_url: -+ if not GRAALPY_VERSION.endswith('-dev'): -+ patches_url = patches_url.replace(VERSION_PARAMETER, GRAALPY_VERSION) -+ else: -+ logger.debug("Skipping versioned GraalPy patch repository on snapshot build") -+ patches_url = None -+ if patches_url and patches_url != DISABLED_PATCHES_URL: -+ logger.info( -+ "Loading GraalPy post-release patch repository from %s. " -+ "This can be controlled with PIP_GRAALPY_PATCHES_URL environment variable. Set to '%s' to disable", -+ patches_url, DISABLED_PATCHES_URL) -+ try: -+ return repository_from_url_or_path(patches_url) -+ except RepositoryException as e: -+ if patches_url == DEFAULT_PATCHES_URL and isinstance(e, RepositoryNotFound): -+ logger.info("No post-release patch repository published yet") -+ else: -+ logger.warning("Failed to load GraalPy patch repository: %s", e) -+ logger.warning("Falling back to bundled GraalPy patch repository") -+ try: -+ return LocalPatchRepository.from_path(DEFAULT_PATCHES_PATH) -+ except RepositoryException as e: -+ logger.warning("Failed to load internal GraalPy patch repository: %s", e) -+ return EmptyRepository() -+ -+ -+def get_patch_repository(): -+ global __PATCH_REPOSITORY -+ if not __PATCH_REPOSITORY: -+ __PATCH_REPOSITORY = create_patch_repository(PATCHES_URL) -+ return __PATCH_REPOSITORY -+ -+ -+def apply_graalpy_patches(filename, location): -+ """ -+ Applies any GraalPy patches to package extracted from 'filename' into 'location'. -+ Note that 'location' must be the parent directory of the package directory itself. -+ For example: /path/to/site-package and not /path/to/site-packages/mypackage. -+ """ -+ if DISABLE_PATCHING: -+ return -+ -+ # we expect filename to be something like "pytest-5.4.2-py3-none-any.whl" -+ archive_name = os.path.basename(filename) -+ name_ver_match = re.match(fr"^(?P.*?)-(?P{VERSION_PATTERN}).*?\.(?Ptar\.gz|tar|whl|zip)$", -+ archive_name, re.VERBOSE | re.I) -+ if not name_ver_match: -+ logger.warning(f"GraalPy warning: could not parse package name, version, or format from {archive_name!r}.\n" -+ "Could not determine if any GraalPy specific patches need to be applied.") -+ return -+ -+ name = name_ver_match.group('name') -+ version = name_ver_match.group('version') -+ suffix = name_ver_match.group('suffix') -+ is_wheel = suffix == "whl" -+ -+ if is_wheel and is_wheel_marked(filename): -+ # We already processed it when building from source -+ return -+ -+ import autopatch_capi -+ import subprocess -+ -+ autopatch_capi.auto_patch_tree(location) -+ -+ logger.info(f"Looking for GraalPy patches for {name}") -+ repository = get_patch_repository() -+ -+ if is_wheel: -+ # patches intended for binary distribution: -+ rule = repository.get_matching_rule(name, version, 'wheel') -+ else: -+ # patches intended for source distribution if applicable -+ rule = repository.get_matching_rule(name, version, 'sdist') -+ if not rule: -+ rule = repository.get_matching_rule(name, version, 'wheel') -+ if rule and (subdir := rule.get('subdir')): -+ # we may need to change wd if we are actually patching a source distribution -+ # with a patch intended for a binary distribution, because in the source -+ # distribution the actual deployed sources may be in a subdirectory (typically "src") -+ location = os.path.join(location, subdir) -+ if rule: -+ if patch := rule.get('patch'): -+ with repository.resolve_patch(patch) as patch_path: -+ if not patch_path: -+ return -+ logger.info(f"Patching package {name} using {patch}") -+ exe = '.exe' if os.name == 'nt' else '' -+ try: -+ subprocess.run([f"patch{exe}", "-f", "-d", location, "-p1", "-i", str(patch_path)], check=True) -+ except FileNotFoundError: -+ logger.warning( -+ "WARNING: GraalPy needs the 'patch' utility to apply compatibility patches. Please install it using your system's package manager.") -+ except subprocess.CalledProcessError: -+ logger.warning(f"Applying GraalPy patch failed for {name}. The package may still work.") -+ except Exception: -+ logger.exception(f"Failed to execute patch utility") -+ elif version_specs := repository.get_suggested_version_specs(name): -+ logger.info("We have patches to make this package work on GraalVM for some version(s).") -+ logger.info("If installing or running fails, consider using one of the versions that we have patches for:") -+ for version_spec in version_specs: -+ logger.info(f'{name} {version_spec}') -+ -+ -+def apply_graalpy_sort_order(sort_key_func): -+ if DISABLE_VERSION_SELECTION: -+ return sort_key_func -+ -+ def wrapper(self, candidate): -+ default_sort_key = sort_key_func(self, candidate) -+ priority = get_patch_repository().get_priority_for_version(candidate.name, str(candidate.version)) -+ return priority, default_sort_key -+ -+ return wrapper -+ -+ -+class AddedSourceLink(Link): -+ def __init__(self, url: str, filename: str): -+ super().__init__(url) -+ self._filename = filename -+ -+ @property -+ def filename(self) -> str: -+ return self._filename -+ -+ -+def get_graalpy_candidates(name): -+ repository = get_patch_repository() -+ candidates = [] -+ for add_source in repository.get_add_sources(name) or []: -+ version = add_source['version'] -+ url = add_source['url'] -+ match = re.search(r'\.(tar\.(?:gz|bz2|xz)|zip|whl)$', urlparse(url).path) -+ assert match, "Couldn't determine URL suffix" -+ suffix = match.group(1) -+ # We need to force the filename to match the usual convention, otherwise we won't find a patch -+ link = AddedSourceLink(url, f'{name}-{version}.{suffix}') -+ candidates.append(InstallationCandidate(name=name, version=version, link=link)) -+ if name == 'graalpy-virtualenv-seeder': -+ link = Link(path_to_url(os.path.join(sys.base_prefix, 'graalpy_virtualenv_seeder'))) -+ candidates.append(InstallationCandidate(name=name, version='0.0.1', link=link)) -+ return candidates -+ -+ -+def mark_wheel(path): -+ if DISABLE_PATCHING: -+ return -+ with zipfile.ZipFile(path, 'a') as z: -+ dist_info = None -+ for name in z.namelist(): -+ if m := re.match(r'([^/]+.dist-info)/', name): -+ dist_info = m.group(1) -+ break -+ assert dist_info, "Cannot find .dist_info in built wheel" -+ marker = f'{dist_info}/{MARKER_FILE_NAME}' -+ with z.open(marker, 'w'): -+ pass -+ -+ -+def is_wheel_marked(path): -+ with zipfile.ZipFile(path) as z: -+ return any(re.match(rf'[^/]+.dist-info/{MARKER_FILE_NAME}$', f) for f in z.namelist()) diff --git a/pip/_internal/utils/unpacking.py b/pip/_internal/utils/unpacking.py index 875e30e..f6562cf 100644 --- a/pip/_internal/utils/unpacking.py @@ -515,7 +155,7 @@ index 875e30e..f6562cf 100644 content_type, ) raise InstallationError(f"Cannot determine archive format of {location}") -+ from pip._internal.utils.graalpy import apply_graalpy_patches ++ from graalpy_pip_extensions import apply_graalpy_patches + apply_graalpy_patches(filename, location) diff --git a/pip/_internal/wheel_builder.py b/pip/_internal/wheel_builder.py index 93f8e1f..32fd4ab 100644 @@ -525,7 +165,7 @@ index 93f8e1f..32fd4ab 100644 import shutil from typing import Iterable, List, Optional, Tuple -+from pip._internal.utils import graalpy ++import graalpy_pip_extensions as graalpy from pip._vendor.packaging.utils import canonicalize_name, canonicalize_version from pip._vendor.packaging.version import InvalidVersion, Version diff --git a/graalpython/lib-graalpython/pip_audit_hook.py b/graalpython/lib-graalpython/pip_audit_hook.py new file mode 100644 index 0000000000..eeaa50bf01 --- /dev/null +++ b/graalpython/lib-graalpython/pip_audit_hook.py @@ -0,0 +1,232 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import re +import sys +from pathlib import Path +from urllib.parse import urlparse +from warnings import warn + +import graalpy_pip_extensions as graalpy + + +# These events are emitted by the GraalOS pip patch; they are not upstream pip audit events. +PIP_CLI_PARSE_ARGS = "pip.cli.parse_args" +PIP_FIND_ALL_CANDIDATES = "pip.package_finder.find_all_candidates" +PIP_REQUIREMENT_SOURCE_READY = "pip.requirement.source_ready" +PIP_WHEEL_INSTALL = "pip.wheel.install" + +GRAALPY_WHEELS_URL = "https://www.graalvm.org/python/wheels/" +PIP_INDEX_URL_ENV_VAR = "PIP_INDEX_URL" +PIP_CACHE_DIR_ENV_VAR = "PIP_CACHE_DIR" +CACHE_DIR_SUFFIX = "-graalpy" + +_PATCHED_SOURCES = set() +_PATCHED_DISTS = set() + + +def _apply_cli_defaults(options, args): + if PIP_INDEX_URL_ENV_VAR not in os.environ and hasattr(options, "extra_index_urls"): + extra_index_urls = getattr(options, "extra_index_urls") + if extra_index_urls is None: + extra_index_urls = [] + elif not isinstance(extra_index_urls, list): + extra_index_urls = list(extra_index_urls) + if GRAALPY_WHEELS_URL not in extra_index_urls: + extra_index_urls.insert(0, GRAALPY_WHEELS_URL) + options.extra_index_urls = extra_index_urls + + if PIP_CACHE_DIR_ENV_VAR not in os.environ and hasattr(options, "cache_dir") and not any(arg == "--cache-dir" or arg.startswith("--cache-dir=") or arg == "--no-cache-dir" for arg in args or ()): + cache_dir = getattr(options, "cache_dir") + if cache_dir: + cache_path = Path(cache_dir) + if cache_path.name and not cache_path.name.endswith(CACHE_DIR_SUFFIX): + options.cache_dir = str(cache_path.with_name(cache_path.name + CACHE_DIR_SUFFIX)) + + +def _candidate_details(candidate): + link = candidate.link + return ( + candidate.name, + str(candidate.version), + { + "url": link.url, + "filename": link.filename, + "comes_from": getattr(link, "comes_from", "GraalPy compatibility patches"), + "requires_python": getattr(link, "requires_python", None), + "yanked_reason": getattr(link, "yanked_reason", None), + }, + ) + + +def _add_graalpy_candidates(project_name, extra_candidates): + extra_candidates.extend(_candidate_details(candidate) for candidate in graalpy.get_graalpy_candidates(project_name)) + + +def _archive_name(link): + if filename := getattr(link, "filename", None): + return os.path.basename(filename) + path = urlparse(str(link)).path + return os.path.basename(path) or os.path.basename(str(link)) + + +def _dist_key(filename): + match = re.match(r"^(?P.*?)-(?P[^-]+).*?\.(?:tar\.(?:gz|bz2|xz)|tar|zip|whl)$", filename, re.I) + if match: + return graalpy.canonicalize_name(match.group("name")), match.group("version") + return None + + +def _apply_graalpy_source_patches(link, source_dir): + if source_dir is None: + return + source_path = Path(source_dir).resolve() + if source_path in _PATCHED_SOURCES: + return + _PATCHED_SOURCES.add(source_path) + + archive_name = _archive_name(link) + if key := _dist_key(archive_name): + _PATCHED_DISTS.add(key) + graalpy.apply_graalpy_patches(archive_name, source_path, warn_suggested_versions=True) + + +def _find_dist_info(root): + for path in root.iterdir(): + if path.is_dir() and path.name.endswith(".dist-info"): + return path + raise RuntimeError("Cannot find .dist-info directory in wheel") + + +def _write_record(root, dist_info): + import base64 + import csv + import hashlib + + record = dist_info / "RECORD" + rows = [] + for path in sorted(p for p in root.rglob("*") if p.is_file()): + rel = path.relative_to(root).as_posix() + if path == record: + rows.append((rel, "", "")) + continue + digest = hashlib.sha256(path.read_bytes()).digest() + encoded = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + rows.append((rel, f"sha256={encoded}", str(path.stat().st_size))) + with record.open("w", encoding="utf-8", newline="") as f: + csv.writer(f).writerows(rows) + + +def _extract_wheel(wheel_path): + import zipfile + from tempfile import TemporaryDirectory + + tmpdir = TemporaryDirectory() + root = Path(tmpdir.name) + with zipfile.ZipFile(wheel_path) as z: + for info in z.infolist(): + target = Path(z.extract(info, root)) + mode = info.external_attr >> 16 + if mode and not info.is_dir(): + target.chmod(mode & 0o777) + return root, tmpdir + + +def _rewrite_wheel(root, wheel_path): + import zipfile + from tempfile import NamedTemporaryFile + + wheel_path = Path(wheel_path) + with NamedTemporaryFile(dir=wheel_path.parent, prefix=wheel_path.name, suffix=".tmp", delete=False) as tmp_file: + tmp_path = Path(tmp_file.name) + try: + with zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as z: + for path in sorted(p for p in root.rglob("*") if p.is_file()): + z.write(path, path.relative_to(root).as_posix()) + os.replace(tmp_path, wheel_path) + finally: + try: + tmp_path.unlink() + except FileNotFoundError: + pass + + +def _mark_extracted_wheel(root): + dist_info = _find_dist_info(root) + (dist_info / graalpy.MARKER_FILE_NAME).touch() + _write_record(root, dist_info) + + +def _apply_graalpy_wheel_patches(wheel_path): + wheel_path = Path(wheel_path) + if graalpy.DISABLE_PATCHING or not wheel_path.is_file() or wheel_path.suffix != ".whl" or graalpy.is_wheel_marked(wheel_path): + return + if _dist_key(wheel_path.name) in _PATCHED_DISTS: + graalpy.mark_wheel(wheel_path) + return + + root, cleanup = _extract_wheel(wheel_path) + try: + graalpy.apply_graalpy_patches(wheel_path, root, warn_suggested_versions=True) + _mark_extracted_wheel(root) + _rewrite_wheel(root, wheel_path) + finally: + cleanup.cleanup() + + +def _audit_hook(event, args): + try: + if event == PIP_CLI_PARSE_ARGS: + options, parsed_args, _verbosity = args + _apply_cli_defaults(options, parsed_args) + elif event == PIP_FIND_ALL_CANDIDATES: + project_name, _candidates, extra_candidates = args + _add_graalpy_candidates(project_name, extra_candidates) + elif event == PIP_REQUIREMENT_SOURCE_READY: + _name, _specifier, link, source_dir, _editable, _verbosity = args + _apply_graalpy_source_patches(link, source_dir) + elif event == PIP_WHEEL_INSTALL: + _name, wheel_path = args + _apply_graalpy_wheel_patches(wheel_path) + except Exception as e: + warn(f"failed to apply pip compatibility patches: {e}") + + +sys.addaudithook(_audit_hook) diff --git a/graalpython/lib-graalpython/pip_hook.py b/graalpython/lib-graalpython/pip_hook.py index 8d466d4b8b..8ef0c011c6 100644 --- a/graalpython/lib-graalpython/pip_hook.py +++ b/graalpython/lib-graalpython/pip_hook.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -38,48 +38,14 @@ # SOFTWARE. import sys -WARNED = False - -def print_version_warning(): - global WARNED - if not WARNED: - from warnings import warn - warn("You are using an untested version of pip. GraalPy " + - "provides patches and workarounds for a number of packages when used with " + - "compatible pip versions. We recommend to stick with the pip version that " + - "ships with this version of GraalPy.", RuntimeWarning) - WARNED = True - - -class PipLoader: - def __init__(self, real_loader): - self.real_loader = real_loader - - def create_module(self, spec): - return self.real_loader.create_module(spec) - - def exec_module(self, module): - self.real_loader.exec_module(module) - if not getattr(module, '__GRAALPY_PATCHED', False): - print_version_warning() - - -class PipImportHook: - @staticmethod - def _wrap_real_spec(fullname, path, target): - for finder in sys.meta_path: - if finder is PipImportHook: - continue - real_spec = finder.find_spec(fullname, path, target) - if real_spec: - real_spec.loader = PipLoader(real_spec.loader) - return real_spec - - @staticmethod - def find_spec(fullname, path, target=None): - if fullname == "pip": - return PipImportHook._wrap_real_spec(fullname, path, target) - - -sys.meta_path.insert(0, PipImportHook) +pip = sys.modules.get("pip") +assert pip, "This should only be loaded after pip is" +if not getattr(pip, "__GRAALPY_PATCHED", False): + from warnings import warn + warn("You are using an unpatched version of pip. GraalPy " + "provides patches and workarounds for a number of packages when used with " + "patched pip versions. We recommend to stick with the pip version that " + "ships with this version of GraalPy or a pip version that implements " + "experimental audit hooks.", RuntimeWarning) + __graalpython__.load_file("pip_audit_hook") diff --git a/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl b/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl index 6e3c73e965..a7902016df 100644 Binary files a/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl and b/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl differ diff --git a/graalpython/lib-python/3/importlib/_bootstrap.py b/graalpython/lib-python/3/importlib/_bootstrap.py index d223bc1a7b..3a2c88600d 100644 --- a/graalpython/lib-python/3/importlib/_bootstrap.py +++ b/graalpython/lib-python/3/importlib/_bootstrap.py @@ -1362,7 +1362,12 @@ def _find_and_load(name, import_): with _ModuleLockManager(name): module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: - return _find_and_load_unlocked(name, import_) + # GraalPy change: we may want to load our pip hook + mod = _find_and_load_unlocked(name, import_) + if name == "pip": + __graalpython__.load_file("pip_hook") + return mod + # End GraalPy change # Optimization: only call _bootstrap._lock_unlock_module() if # module.__spec__._initializing is True. diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index f70e224544..bf47948043 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -1338,7 +1338,7 @@ "./META-INF/resources///Lib/venv/scripts/nt/graalpy.exe": "dependency:python-venvlauncher", "./META-INF/resources///Lib/venv/scripts/nt/python.exe": "dependency:python-venvlauncher", "./META-INF/resources///include/": "dependency:graalpy-pyconfig/-//pyconfig.h", - "./META-INF/resources///lib-graalpython/modules/": "dependency:graalpy-pyconfig/-//.py", + "./META-INF/resources///Lib/": "dependency:graalpy-pyconfig/-//.py", }, }, }, @@ -1350,7 +1350,7 @@ ], "./META-INF/resources///lib/python/venv/scripts/macos/graalpy": "dependency:python-macos-launcher", "./META-INF/resources///include/python/": "dependency:graalpy-pyconfig/-//pyconfig.h", - "./META-INF/resources///lib/graalpy/modules/": "dependency:graalpy-pyconfig/-//.py", + "./META-INF/resources///lib/python/": "dependency:graalpy-pyconfig/-//.py", } } }, @@ -1361,7 +1361,7 @@ "dependency:GRAALPYTHON_NATIVE_LIBS///*", ], "./META-INF/resources///include/python/": "dependency:graalpy-pyconfig/-//pyconfig.h", - "./META-INF/resources///lib/graalpy/modules/": "dependency:graalpy-pyconfig/-//.py", + "./META-INF/resources///lib/python/": "dependency:graalpy-pyconfig/-//.py", }, }, },