diff --git a/evaluation_script/VENDOR_GUIDE.md b/evaluation_script/VENDOR_GUIDE.md new file mode 100644 index 000000000..c6c4a2f96 --- /dev/null +++ b/evaluation_script/VENDOR_GUIDE.md @@ -0,0 +1,29 @@ +# Vendoring third-party packages for evaluation + +Place any third-party packages required by your evaluation script into `evaluation_script/vendor/` so the evaluation can run without network access or installing outside the repository. + +Options: + +- Vendor a package directory: + - Put the package folder (containing `__init__.py`) under `evaluation_script/vendor//`. + - Example: `evaluation_script/vendor/zhipuai/__init__.py` + +- Vendor a local wheel or tarball: + - Place the `.whl` or `.tar.gz` file into `evaluation_script/vendor/` and in your CI or build step install it into `vendor/` (offline install). The `__init__.py` will prefer the contents of `vendor/` on `sys.path`. + +- Document expected packages: + - Add `evaluation_script/local_requirements.txt` with one package per line (optionally with versions) so missing packages can be detected and warned about at import time. + +Why this approach? + +- Keeps all code and dependencies inside the repository. +- Avoids changing the worker or host Python environment. +- Makes evaluations reproducible and network-independent. + +How to use: + +- Add vendored packages to `evaluation_script/vendor/`. +- (Optional) add package names to `evaluation_script/local_requirements.txt`. +- The updated `evaluation_script/__init__.py` will prepend `vendor/` to `sys.path` automatically. + +If you want help vendoring a specific package (for example `zhipuai`), tell me whether you can provide the package source or wheel and I will add it under `evaluation_script/vendor/` for you. diff --git a/evaluation_script/__init__.py b/evaluation_script/__init__.py index 10a8ebbf2..69c1638e7 100644 --- a/evaluation_script/__init__.py +++ b/evaluation_script/__init__.py @@ -1,56 +1,64 @@ """ -# Q. How to install custom python pip packages? +Vendor-friendly `__init__` for evaluation scripts. -# A. Uncomment the below code to install the custom python packages. +Goal: +- Prefer packages that are included inside the repository under + `evaluation_script/vendor/` so evaluation does NOT require network access + or installs outside the repo. + +Usage: +- To include a third-party package, add the package directory or a + locally-downloaded wheel into `evaluation_script/vendor/`. +- Optionally create `evaluation_script/local_requirements.txt` with + package names (one per line) to document which packages are expected. + +Behavior: +- If `vendor/` exists it will be prepended to `sys.path` so imports + like `from zhipuai import ZhipuAI` will resolve when `zhipuai` is + present under `evaluation_script/vendor/`. +- If packages listed in `local_requirements.txt` are missing from the + vendor folder, a warning is printed (no network install is attempted). + +This approach ensures all code required for evaluation can live inside +the repo and avoids modifying the host Python environment. +""" -import os -import subprocess import sys +import importlib.util from pathlib import Path -def install(package): - # Install a pip python package - - # Args: - # package ([str]): Package name with version - try: - subprocess.run([sys.executable,"-m","pip","install",package]) - except subprocess.CalledProcessError as e: - print(f"Error occurred while installing {package}: {e.stderr}") - except FileNotFoundError: - print("Error: Pip is not found. make sure you have pip installed.") - except PermissionError: - print("Error: Permission denied. ") - - -def install_local_package(folder_name): - # Install a local python package - - # Args: - # folder_name ([str]): name of the folder placed in evaluation_script/ - - - try: - subprocess.run([ - sys.executable, - "-m", - "pip", - "install", - os.path.join(str(Path(__file__).parent.absolute()), folder_name) - ], capture_output=True, text=True, check=True) - print(f"Successfully installed local package from {folder_name}.") - except subprocess.CalledProcessError as e: - print(f"Error occurred while installing local package from {folder_name}: {e.stderr}") - except FileNotFoundError: - print("Error: Pip not found.") - except PermissionError: - print("Error: Permission denied. ") - -install("shapely==1.7.1") -install("requests==2.25.1") - -install_local_package("package_folder_name") +VENDOR_DIR = Path(__file__).parent / "vendor" -""" +def add_vendor_to_sys_path(): + if VENDOR_DIR.exists() and str(VENDOR_DIR) not in sys.path: + sys.path.insert(0, str(VENDOR_DIR)) + print(f"[evaluation_script] Added vendor directory to sys.path: {VENDOR_DIR}") + return True + return False + +def is_importable(module_name: str) -> bool: + return importlib.util.find_spec(module_name) is not None + +def check_local_requirements(): + req_file = Path(__file__).parent / "local_requirements.txt" + missing = [] + if req_file.exists(): + for line in req_file.read_text().splitlines(): + pkg = line.strip() + if not pkg or pkg.startswith("#"): + continue + # only check module base name (before any ==) + mod_name = pkg.split("==")[0].strip() + if not is_importable(mod_name): + missing.append(pkg) + return missing + +# Add vendor path early so downstream imports can find vendored packages. +add_vendor_to_sys_path() + +missing = check_local_requirements() +if missing: + print("[evaluation_script] Warning: The following packages listed in local_requirements.txt were not found in evaluation_script/vendor/:\n " + "\n ".join(missing)) + print("[evaluation_script] To avoid network installs, place package directories or wheels into evaluation_script/vendor/") from .main import evaluate diff --git a/evaluation_script/test_vendor_import.py b/evaluation_script/test_vendor_import.py new file mode 100644 index 000000000..2a4a09a26 --- /dev/null +++ b/evaluation_script/test_vendor_import.py @@ -0,0 +1,16 @@ +import importlib.util +import importlib +import sys + +print('Before importing evaluation_script: zhipuai importable =', importlib.util.find_spec('zhipuai') is not None) + +# Importing the package triggers evaluation_script/__init__.py which adds vendor/ to sys.path +import evaluation_script + +print('After importing evaluation_script: zhipuai importable =', importlib.util.find_spec('zhipuai') is not None) + +from zhipuai import ZhipuAI, greet + +obj = ZhipuAI() +print('ZhipuAI.name =', obj.name) +print('greet() =', greet()) diff --git a/evaluation_script/vendor/zhipuai/__init__.py b/evaluation_script/vendor/zhipuai/__init__.py new file mode 100644 index 000000000..8f9d7c97d --- /dev/null +++ b/evaluation_script/vendor/zhipuai/__init__.py @@ -0,0 +1,9 @@ +class ZhipuAI: + def __init__(self): + self.name = "dummy zhipuai" + + def info(self): + return "This is a dummy vendored zhipuai package for testing." + +def greet(): + return "hello from zhipuai"