diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 056dd11b..1f5c3fda 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -31,6 +31,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' steps: - uses: actions/checkout@v4 @@ -78,6 +79,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' steps: - uses: actions/checkout@v4 @@ -134,3 +136,22 @@ jobs: # # - name: Run linters # run: tox -e lint + + check-readme: + name: Check README + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install dependencies + run: python -m pip install --upgrade tox tox-gh-actions + + - name: Run README check + run: tox -e check-readme + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 724452bc..37234781 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,30 +1,32 @@ --- name: Release -'on': +"on": release: types: [published] permissions: - contents: read # required for github.ref to be set + contents: read # required for github.ref to be set jobs: pypi-publish: name: Upload release to PyPI runs-on: ubuntu-latest environment: - name: pypi - url: https://pypi.org/p/alibuild + name: release + url: https://pypi.org/p/bits # TODO: update with actual package name permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - # retrieve your distributions here - - name: Install build dependencies - run: python -m pip install --upgrade setuptools build pip - - name: Build the Python distribution - run: python -m build - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install build dependencies + run: python -m pip install --upgrade setuptools build pip + - name: Build the Python distribution + run: python -m build + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 brew: name: Update Homebrew formula @@ -39,15 +41,14 @@ jobs: - name: Update bits formula with new version run: | set -x - json=$(curl -fSsL "https://pypi.org/pypi/bits/${GITHUB_REF#refs/tags/}/json") - sha=$(echo "$json" | jq -r '.urls[0].digests.sha256') - url=$(echo "$json" | jq -r '.urls[0].url') + json=$(curl -fSsL "https://pypi.org/pypi/bits/${GITHUB_REF#refs/tags/}/json") # TODO: update with actual package name + sha=$(echo "$json" | jq -r '.urls.[] | select(.filename | endswith(".tar.gz")) | .digests.sha256') + url=$(echo "$json" | jq -r '.urls.[] | select(.filename | endswith(".tar.gz")) | .url') # Replace keys with two leading spaces only, so we get the toplevel # ones applying to bits, not those for dependencies. sed -i.bak " s|^ url .*| url \"$url\"|; s|^ sha256 .*| sha256 \"$sha\"|; - s|^ version .*| version \"${GITHUB_REF#refs/tags/v}\"| " "$(brew formula bits)" rm -f "$(brew formula bits).bak" - name: Regenerate bits formula @@ -70,6 +71,6 @@ jobs: git add bits.rb git commit -m "Update bits to ${GITHUB_REF#refs/tags/}" git show - git push "https://bits:$GITHUB_TOKEN@github.com/bitsorg/homebrew-system-deps" master + git push "https://bits:$GITHUB_TOKEN@github.com/bits/homebrew-system-deps" master # TODO: update user and repo env: GITHUB_TOKEN: ${{ secrets.homebrew_repo_token }} diff --git a/README.rst b/README.rst index fd12a08c..c6e2c8a2 100644 --- a/README.rst +++ b/README.rst @@ -35,14 +35,22 @@ Pre-requisites If you are using bits directly from git clone, you should make sure you have the dependencies installed. The easiest way to do this is to run:: + # Optional, make a venv so the dependencies are not installed globally + python -m venv .venv + source .venv/bin/activate pip install -e . -For developers -========== +Contributing +============ + If you want to contribute to bits, you can run the tests with:: + # Optional, make a venv so the dependencies are not installed globally + python -m venv .venv + source .venv/bin/activate + pip install -e .[test] # Only needed once tox @@ -56,6 +64,13 @@ You can also run only the unit tests (it's a lot faster than the full suite) wit To run the documentation locally, you can use:: + # Optional, make a venv so the dependencies are not installed globally + python -m venv .venv + source .venv/bin/activate + + # Install dependencies for the docs, check pyproject.toml for more info pip install -e .[docs] + + # Run the docs cd docs mkdocs serve diff --git a/bits_helpers/args.py b/bits_helpers/args.py index b6b2d755..ab083818 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -262,6 +262,8 @@ def doParseArgs(): help=("Assume we're not building %(metavar)s and all its (unique) dependencies. " "You can specify this option multiple times or separate multiple arguments " "with commas.")) + doctor_parser.add_argument("-e", dest="environment", action="append", default=[], + help="KEY=VALUE binding to add to the build environment. May be specified multiple times.") doctor_system = doctor_parser.add_mutually_exclusive_group() doctor_system.add_argument("--always-prefer-system", dest="preferSystem", action="store_true", diff --git a/bits_helpers/build.py b/bits_helpers/build.py index e75fda2b..2c006aba 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -60,7 +60,7 @@ def update_repo(package, git_prompt): specs[package]["scm"] = Git() if specs[package]["is_devel_pkg"]: specs[package]["source"] = os.path.join(os.getcwd(), specs[package]["package"]) - if exists(os.path.join(specs[package]["source"], ".sl")): + if exists(os.path.join(specs[package]["source"], ".sl")) or exists(os.path.join(specs[package]["source"], ".git/sl")): specs[package]["scm"] = Sapling() updateReferenceRepoSpec(args.referenceSources, package, specs[package], fetch=args.fetchRepos, allowGitPrompt=git_prompt) @@ -515,7 +515,10 @@ def doBuild(args, parser): install_wrapper_script("git", workDir) - with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker: + extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)} + extra_env.update(dict([e.partition('=')[::2] for e in args.environment])) + + with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker: def performPreferCheckWithTempDir(pkg, cmd): with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir: return getstatusoutput_docker(cmd, cwd=temp_dir) @@ -1150,12 +1153,13 @@ def performPreferCheckWithTempDir(pkg, cmd): if args.docker: build_command = ( "docker run --rm --entrypoint= --user $(id -u):$(id -g) " - "-v {workdir}:/sw -v {scriptDir}/build.sh:/build.sh:ro " + "-v {workdir}:/sw -v{configDir}:/alidist:ro -v {scriptDir}/build.sh:/build.sh:ro " "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} " - "-e WORK_DIR_OVERRIDE=/sw {extraArgs} {image} bash -ex /build.sh" + "-e WORK_DIR_OVERRIDE=/sw -e BITS_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} {image} bash -ex /build.sh" ).format( image=quote(args.dockerImage), workdir=quote(abspath(args.workDir)), + configDir=quote(abspath(args.configDir)), scriptDir=quote(scriptDir), extraArgs=" ".join(map(quote, args.docker_extra_args)), additionalEnv=" ".join( diff --git a/bits_helpers/build_template.sh b/bits_helpers/build_template.sh index 616d003e..6da62397 100644 --- a/bits_helpers/build_template.sh +++ b/bits_helpers/build_template.sh @@ -16,6 +16,7 @@ cpath=$(which gcc) function hash() { true; } export WORK_DIR="${WORK_DIR_OVERRIDE:-%(workDir)s}" +export BITS_CONFIG_DIR="${BITS_CONFIG_DIR_OVERRIDE:-%(configDir)s}" # Insert our own wrapper scripts into $PATH, patched to use the system OpenSSL, # instead of the one we build ourselves. @@ -108,9 +109,9 @@ unset DYLD_LIBRARY_PATH EOF cd "$BUILDROOT" -ln -snf $PKGHASH "${BUILDROOT}-latest" +ln -snf "$PKGHASH" "$ALIBUILD_BUILD_WORK_DIR/BUILD/$PKGNAME-latest" if [[ $DEVEL_PREFIX ]]; then - ln -snf $PKGHASH "${BUILDROOT}-latest-$DEVEL_PREFIX" + ln -snf "$PKGHASH" "$ALIBUILD_BUILD_WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX" fi cd "$BUILDDIR" diff --git a/bits_helpers/cmd.py b/bits_helpers/cmd.py index 65adf6e7..ac5a0119 100644 --- a/bits_helpers/cmd.py +++ b/bits_helpers/cmd.py @@ -78,25 +78,42 @@ class DockerRunner: instead. """ - def __init__(self, docker_image, docker_run_args=()) -> None: + def __init__(self, docker_image, docker_run_args=(), extra_env={}, extra_volumes=[]) -> None: self._docker_image = docker_image self._docker_run_args = docker_run_args self._container = None + self._extra_env = extra_env + self._extra_volumes = extra_volumes def __enter__(self): if self._docker_image: # "sleep inf" pauses forever, until we kill it. - cmd = ["docker", "run", "--detach", "--rm", "--entrypoint="] + envOpts = [] + volumes = [] + for env in self._extra_env.items(): + envOpts.append("-e") + envOpts.append(f"{env[0]}={env[1]}") + for v in self._extra_volumes: + volumes.append("-v") + volumes.append(v) + cmd = ["docker", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint="] cmd += self._docker_run_args cmd += [self._docker_image, "sleep", "inf"] self._container = getoutput(cmd).strip() def getstatusoutput_docker(cmd, cwd=None): if self._container is None: - return getstatusoutput("{} -c {}".format(BASH, quote(cmd)), cwd=cwd) - return getstatusoutput("docker container exec {} bash -c {}" - .format(quote(self._container), quote(cmd)), - cwd=cwd) + command_prefix="" + if self._extra_env: + command_prefix="env " + " ".join("{}={}".format(k, quote(v)) for (k,v) in self._extra_env.items()) + " " + return getstatusoutput("{}{} -c {}".format(command_prefix, BASH, quote(cmd)) + , cwd=cwd) + envOpts = [] + for env in self._extra_env.items(): + envOpts.append("-e") + envOpts.append("{}={}".format(env[0], env[1])) + exec_cmd = ["docker", "container", "exec"] + envOpts + [self._container, "bash", "-c", cmd] + return getstatusoutput(exec_cmd, cwd=cwd) return getstatusoutput_docker diff --git a/bits_helpers/deps.py b/bits_helpers/deps.py index a16aa80e..6bff03fc 100644 --- a/bits_helpers/deps.py +++ b/bits_helpers/deps.py @@ -4,7 +4,7 @@ from bits_helpers.utilities import parseDefaults, readDefaults, getPackageList, validateDefaults from bits_helpers.cmd import DockerRunner, execute from tempfile import NamedTemporaryFile -from os import remove +from os import remove, path def doDeps(args, parser): @@ -16,7 +16,14 @@ def doDeps(args, parser): specs = {} defaultsReader = lambda: readDefaults(args.configDir, args.defaults, parser.error, args.architecture, args.xdefaults) (err, overrides, taps) = parseDefaults(args.disable, defaultsReader, debug) - with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker: + + extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else path.abspath(args.configDir)} + extra_env.update(dict([e.partition('=')[::2] for e in args.environment])) + + with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker: + def performCheck(pkg, cmd): + return getstatusoutput_docker(cmd) + systemPackages, ownPackages, failed, validDefaults = \ getPackageList(packages = [args.package], specs = specs, @@ -26,8 +33,8 @@ def doDeps(args, parser): architecture = args.architecture, disable = args.disable, defaults = args.defaults, - performPreferCheck = lambda pkg, cmd: getstatusoutput_docker(cmd), - performRequirementCheck = lambda pkg, cmd: getstatusoutput_docker(cmd), + performPreferCheck = performCheck, + performRequirementCheck = performCheck, performValidateDefaults = lambda spec: validateDefaults(spec, args.defaults), overrides = overrides, taps = taps, diff --git a/bits_helpers/doctor.py b/bits_helpers/doctor.py index 0f5a5b7e..ddee9ae0 100644 --- a/bits_helpers/doctor.py +++ b/bits_helpers/doctor.py @@ -22,7 +22,8 @@ def checkPreferSystem(spec, cmd, homebrew_replacement, getstatusoutput_docker): debug("Package %s can only be managed via bits.", spec["package"]) return (1, "") cmd = homebrew_replacement + cmd - err, out = getstatusoutput_docker(cmd) + with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{spec['package']}_") as temp_dir: + err, out = getstatusoutput_docker(cmd, cwd=temp_dir) if not err: success("Package %s will be picked up from the system.", spec["package"]) for x in out.split("\n"): @@ -40,7 +41,8 @@ def checkRequirements(spec, cmd, homebrew_replacement, getstatusoutput_docker): debug("Package %s is not a system requirement.", spec["package"]) return (0, "") cmd = homebrew_replacement + cmd - err, out = getstatusoutput_docker(cmd) + with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{spec['package']}_") as temp_dir: + err, out = getstatusoutput_docker(cmd, cwd=temp_dir) if not err: success("Required package %s will be picked up from the system.", spec["package"]) debug("%s", cmd) @@ -83,7 +85,11 @@ def doDoctor(args, parser): # Decide if we can use homebrew. If not, we replace it with "true" so # that we do not get spurious messages on linux homebrew_replacement = "" - with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker: + + extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)} + extra_env.update(dict([e.partition('=')[::2] for e in args.environment])) + + with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker: err, output = getstatusoutput_docker("type c++") if err: warning("Unable to find system compiler.\n" @@ -140,10 +146,10 @@ def performValidateDefaults(spec): error("%s", msg) return (ok,msg,valid) - with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker: - def performPreferCheckWithTempDir(pkg, cmd): - with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir: - return getstatusoutput_docker(cmd, cwd=temp_dir) + extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)} + extra_env.update(dict([e.partition('=')[::2] for e in args.environment])) + + with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker: fromSystem, own, failed, validDefaults = \ getPackageList(packages = packages, specs = specs, @@ -153,8 +159,8 @@ def performPreferCheckWithTempDir(pkg, cmd): architecture = args.architecture, disable = args.disable, defaults = args.defaults, - performPreferCheck = performPreferCheckWithTempDir, - performRequirementCheck = performPreferCheckWithTempDir, + performPreferCheck = lambda pkg, cmd: checkPreferSystem(pkg, cmd, homebrew_replacement, getstatusoutput_docker), + performRequirementCheck = lambda pkg, cmd: checkRequirements(pkg, cmd, homebrew_replacement, getstatusoutput_docker), performValidateDefaults = performValidateDefaults, overrides = overrides, taps = taps, diff --git a/bits_helpers/git.py b/bits_helpers/git.py index c0e86765..000b7e2d 100644 --- a/bits_helpers/git.py +++ b/bits_helpers/git.py @@ -68,7 +68,7 @@ def checkoutCmd(self, tag): return ["checkout", "-f", tag] def fetchCmd(self, remote, *refs): - return ["fetch", "-f"] + clone_speedup_options() + [remote, *refs] + return ["fetch", "-f", "--prune"] + clone_speedup_options() + [remote, *refs] def setWriteUrlCmd(self, url): return ["remote", "set-url", "--push", "origin", url] diff --git a/bits_helpers/init.py b/bits_helpers/init.py index c9d68c5b..d5d2027b 100644 --- a/bits_helpers/init.py +++ b/bits_helpers/init.py @@ -3,6 +3,7 @@ from bits_helpers.log import debug, error, warning, banner, info from bits_helpers.log import dieOnError from bits_helpers.workarea import updateReferenceRepoSpec +from bits_helpers.cmd import getstatusoutput from os.path import join import os.path as path @@ -54,7 +55,7 @@ def doInit(args): architecture="", disable=[], defaults=args.defaults, - performPreferCheck=lambda *x, **y: (1, ""), + performPreferCheck=lambda pkg, cmd: getstatusoutput(["bash", "-c", cmd]), performRequirementCheck=lambda *x, **y: (0, ""), performValidateDefaults=lambda spec : validateDefaults(spec, args.defaults), overrides=overrides, diff --git a/bits_helpers/log.py b/bits_helpers/log.py index 35291ccd..6f0e3434 100644 --- a/bits_helpers/log.py +++ b/bits_helpers/log.py @@ -53,14 +53,21 @@ def log_current_package(package, main_package, specs, devel_prefix) -> None: class ProgressPrint: - def __init__(self, begin_msg="") -> None: + def __init__(self, begin_msg="", min_interval=0.) -> None: self.count = -1 self.lasttime = 0 self.STAGES = ".", "..", "...", "....", ".....", "....", "...", ".." self.begin_msg = begin_msg self.percent = -1 + self.min_interval = min_interval + self.last_update = 0 def __call__(self, txt, *args) -> None: + now = time.time() + if (now - self.last_update) < self.min_interval: + return + self.last_update = now + if logger.level <= logging.DEBUG or not sys.stdout.isatty(): debug(txt, *args) return diff --git a/bits_helpers/sl.py b/bits_helpers/sl.py index 5afee89e..3ac6cb2e 100644 --- a/bits_helpers/sl.py +++ b/bits_helpers/sl.py @@ -20,9 +20,22 @@ def checkedOutCommitName(self, directory): return sapling(("whereami", ), directory) def branchOrRef(self, directory): - # Format is [+] - identity = sapling(("identify", ), directory) - return identity.split(" ")[-1] + # Format is * branch ref or nothing + err, output = getstatusoutput("""\ + set -e +x + sl -R {directory} bookmark -r . 2>/dev/null | grep -- "*" + """.format( + directory=quote(directory), + ), timeout=SL_COMMAND_TIMEOUT_SEC) + if err > 1: + raise SCMError("Error {} from sl bookmark -r . : {}".format(err, output)) + # We use "none" to indicate there are no bookmarks. This means + # that a devel package will act as a single branch, regardless of where we are. + if not output.strip(): + return "none" + # If a bookmark is there, we use it to determine that we should rebuild + # when we move to it + return output.split(" ")[2] def exec(self, *args, **kwargs): return sapling(*args, **kwargs) diff --git a/bits_helpers/sync.py b/bits_helpers/sync.py index 311e8919..c40192a0 100644 --- a/bits_helpers/sync.py +++ b/bits_helpers/sync.py @@ -7,7 +7,9 @@ import sys import time import requests +from concurrent.futures import ThreadPoolExecutor, as_completed from requests.exceptions import RequestException +from urllib.parse import quote from bits_helpers.cmd import execute from bits_helpers.log import debug, info, error, dieOnError, ProgressPrint @@ -59,6 +61,7 @@ def __init__(self, remoteStore, architecture, workdir, insecure) -> None: def getRetry(self, url, dest=None, returnResult=False, log=True, session=None, progress=debug): get = session.get if session is not None else requests.get + url = quote(url, safe=":/") for i in range(0, self.httpConnRetries): if i > 0: pauseSec = self.httpBackoff * (2 ** (i - 1)) @@ -178,7 +181,7 @@ def fetch_tarball(self, spec) -> None: destPath = os.path.join(self.workdir, store_path, use_tarball) if not os.path.isfile(destPath): # do not download twice progress = ProgressPrint("Downloading tarball for %s@%s" % - (spec["package"], spec["version"])) + (spec["package"], spec["version"]), min_interval=5.0) progress("[0%%] Starting download of %s", use_tarball) # initialise progress bar self.getRetry("/".join((self.remoteStore, store_path, use_tarball)), destPath, session=session, progress=progress) @@ -357,7 +360,7 @@ def fetch_symlinks(self, spec) -> None: # Create the dummy tarball, if it does not exists test -f "{workDir}/{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash/$tarball" && continue mkdir -p "{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}" - find "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version" ! -name etc -maxdepth 1 -mindepth 1 -exec ln -sf {} "{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}/" \; + find "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version" ! -name etc -maxdepth 1 -mindepth 1 -exec ln -sf {} "{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}/" \\; cp -fr "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version/etc" "{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}/etc" mkdir -p "{workDir}/TARS/{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash" tar -C "{workDir}/INSTALLROOT/$pkg_hash" -czf "{workDir}/TARS/{architecture}/store/${{pkg_hash:0:2}}/$pkg_hash/$tarball" . @@ -552,7 +555,7 @@ def fetch_tarball(self, spec) -> None: for tarball in self._s3_listdir(store_path): debug("Fetching tarball %s", tarball) progress = ProgressPrint("Downloading tarball for %s@%s" % - (spec["package"], spec["version"])) + (spec["package"], spec["version"]), min_interval=5.0) progress("[0%%] Starting download of %s", tarball) # initialise progress bar # Create containing directory locally. (exist_ok= is python3-specific.) os.makedirs(os.path.join(self.workdir, store_path), exist_ok=True) @@ -680,15 +683,45 @@ def upload_symlinks_and_tarball(self, spec) -> None: # Second, upload dist symlinks. These should be in place before the main # tarball, to avoid races in the publisher. - for link_dir, symlinks in dist_symlinks.items(): - for link_key, hash_path in symlinks: - self.s3.put_object(Bucket=self.writeStore, - Key=link_key, - Body=os.fsencode(hash_path), - ACL="public-read", - WebsiteRedirectLocation=hash_path) - debug("Uploaded %d dist symlinks to S3 from %s", - len(symlinks), link_dir) + start_time = time.time() + total_symlinks = 0 + + # Limit concurrency to avoid overwhelming S3 with too many simultaneous requests + max_workers = min(32, (len(dist_symlinks) * 10) or 1) + + def _upload_single_symlink(link_key, hash_path): + self.s3.put_object(Bucket=self.writeStore, + Key=link_key, + Body=os.fsencode(hash_path), + ACL="public-read", + WebsiteRedirectLocation=hash_path) + return link_key + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_info = {} + for link_dir, symlinks in dist_symlinks.items(): + for link_key, hash_path in symlinks: + future = executor.submit(_upload_single_symlink, link_key, hash_path) + future_to_info[future] = (link_dir, link_key) + total_symlinks += 1 + + dir_counts = {link_dir: 0 for link_dir in dist_symlinks.keys()} + for future in as_completed(future_to_info): + link_dir, link_key = future_to_info[future] + try: + future.result() + dir_counts[link_dir] += 1 + except Exception as e: + error("Failed to upload symlink %s: %s", link_key, e) + raise + + for link_dir, count in dir_counts.items(): + if count > 0: + debug("Uploaded %d dist symlinks to S3 from %s", count, link_dir) + + end_time = time.time() + debug("Uploaded %d dist symlinks in %.2f seconds", + total_symlinks, end_time - start_time) self.s3.upload_file(Bucket=self.writeStore, Key=tar_path, Filename=os.path.join(self.workdir, tar_path)) diff --git a/bits_helpers/utilities.py b/bits_helpers/utilities.py index 1e0f5ee5..752a9171 100644 --- a/bits_helpers/utilities.py +++ b/bits_helpers/utilities.py @@ -246,7 +246,7 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, pl distribution = distribution.lower() # If platform.dist does not return something sensible, # let's try with /etc/os-release - if distribution not in ["ubuntu", "red hat enterprise linux", "redhat", "centos", "almalinux", "rockylinux"] and hasOsRelease: + if distribution not in ["ubuntu", "red hat enterprise linux", "redhat", "centos", "almalinux", "rocky linux"] and hasOsRelease: for x in osReleaseLines: key, is_prop, val = x.partition("=") if not is_prop: @@ -266,7 +266,7 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, pl if version in debian_ubuntu: distribution = "ubuntu" version = debian_ubuntu[version] - elif distribution in ["redhat", "red hat enterprise linux", "centos", "almalinux", "rockylinux"]: + elif distribution in ["redhat", "red hat enterprise linux", "centos", "almalinux", "rocky linux"]: distribution = "slc" processor = platformProcessor diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 6c4730db..00000000 --- a/docs/README.md +++ /dev/null @@ -1 +0,0 @@ -Read the documentation here: diff --git a/docs/docs/reference.md b/docs/docs/reference.md index aac8c23c..0eed8a7b 100644 --- a/docs/docs/reference.md +++ b/docs/docs/reference.md @@ -79,13 +79,16 @@ The following entries are optional in the header: - `source`: URL of a Git repository from which the source is downloaded. It's good practice to make sure that they are already patched, so that you can easily point to the actual sources used by the software. + - `write_repo`: in case the repository URL to be used for developing is different from the `source`, set this key. It is used by `bits init`, which will initialise your local repository with the `upstream` remote pointing at this URL instead of the one in `source`. + - `tag`: git reference in the above mentioned repository which points to the software to be built. This can be a tag name, a branch name or a commit hash. + - `env`: dictionary whose key-value pairs are environment variables to be set after the recipe is built. The values are interpreted as the contents of a double-quoted shell string, so you can reference other environment variables @@ -101,6 +104,7 @@ The following entries are optional in the header: intended to be used to point to build products of the current recipe. If you need to set an environment variable for use in the recipe, use `export VARIABLE=value` in the recipe body. + - `prepend_path`: dictionary whose key-value pairs are an environment variable name and a path to be prepended to it, as it happens in `LD_LIBRARY_PATH`. This happens only after the package declaring the `prepend_path` in question @@ -116,9 +120,11 @@ The following entries are optional in the header: will result in prepending `$FOO_ROOT/binexec/foobar` to `$PATH`, and both `$FOO_ROOT/sub/lib` and `lib64` to `LD_LIBRARY_PATH`. + - `append_path`: same as `prepend_path` but paths are appended rather than prepended. Like `append_path` and `env`, this **does not** affect the environment of the current recipe. + - `requires`: a list of run-time dependencies for the package, *e.g.*: ```yaml @@ -145,6 +151,7 @@ The following entries are optional in the header: will make sure that `IgProf` is only built on platforms whose name does not begin with `osx`. + - `build_requires`: a list of build-time dependencies for the package. Like `requires`, these packages will be built before the current package is built. @@ -154,8 +161,10 @@ The following entries are optional in the header: `requires`: for instance, RPMs produced for a package won't depend on its `build_requires`, and `alibuild-generate-module` won't pull in build requirements' modulefiles. + - `force_rebuild`: set it to `true` to force re-running the build recipe every time you invoke bits on it. + - `prefer_system_check`: a script which is used to determine whether or not the system equivalent of the package can be used. See also `prefer_system`. If the `--no-system` option is specified, this key is not @@ -170,6 +179,7 @@ The following entries are optional in the header: does not match, the check is skipped and the recipe is run. Using the switch `--always-prefer-system` runs the check always (even when the regular expression for the architecture does not match). + - `relocate_paths`: a list of toplevel paths scanned recursively to perform relocation of executables and dynamic libraries **on macOS only**. If not specified, defaults to `bin`, `lib` and `lib64`. diff --git a/docs/docs/user.md b/docs/docs/user.md index b474984d..c51f8052 100644 --- a/docs/docs/user.md +++ b/docs/docs/user.md @@ -9,15 +9,17 @@ For a quick start introduction, please look [here](quick.md). ``` bits build [-h] [--defaults DEFAULT] - [-a ARCH] [--force-unknown-architecture] - [-z [DEVELPREFIX]] [-e ENVIRONMENT] [-j JOBS] [-u] - [--no-local PKGLIST] [--disable PACKAGE] - [--always-prefer-system | --no-system] - [--docker] [--docker-image IMAGE] [--docker-extra-args ARGLIST] [-v VOLUMES] - [--no-remote-store] [--remote-store STORE] [--write-store STORE] [--insecure] - [-C DIR] [-w WORKDIR] [-c CONFIGDIR] [--reference-sources MIRRORDIR] - [--aggressive-cleanup] [--no-auto-cleanup] - PACKAGE [PACKAGE ...] + [-a ARCH] [--force-unknown-architecture] + [-z [DEVELPREFIX]] [-e ENVIRONMENT] [-j JOBS] [-u] + [--no-local PKGLIST] [--force-tracked] [--disable PACKAGE] + [--force-rebuild PACKAGE] [--annotate PACKAGE=COMMENT] + [--only-deps] [--plugin PLUGIN] + [--always-prefer-system | --no-system] + [--docker] [--docker-image IMAGE] [--docker-extra-args ARGLIST] [-v VOLUMES] + [--no-remote-store] [--remote-store STORE] [--write-store STORE] [--insecure] + [-C DIR] [-w WORKDIR] [-c CONFIGDIR] [--reference-sources MIRRORDIR] + [--aggressive-cleanup] [--no-auto-cleanup] + PACKAGE [PACKAGE ...] ``` - `PACKAGE`: One of the packages in `CONFIGDIR`. May be specified multiple @@ -39,7 +41,18 @@ bits build [-h] [--defaults DEFAULT] given. - `--no-local PKGLIST`: Do not pick up the following packages from a local checkout. `PKGLIST` is a comma-separated list. +- `--force-tracked`: Do not pick up any packages from a local checkout. - `--disable PACKAGE`: Do not build `PACKAGE` and all its (unique) dependencies. +- `--force-rebuild PACKAGE`: Always rebuild the specified packages from scratch, + even if they were built before. Has the same effect as adding + `force_rebuild: true` to the recipe. May be specified multiple times or + separate multiple arguments with commas. +- `--annotate PACKAGE=COMMENT`: Store `COMMENT` in the build metadata for + `PACKAGE`. The comment will only be stored if the package is compiled or + downloaded during this run. May be specified multiple times. +- `--only-deps`: Only build dependencies, not the main package. Useful for + populating a build cache. +- `--plugin PLUGIN`: Plugin to use for the build. Default is `legacy`. - `--always-prefer-system`: Always use system packages when compatible. - `--no-system`: Never use system packages, even if compatible. @@ -102,8 +115,8 @@ provides tarballs for the most common supported architectures. ## Using precompiled packages -By running bits with no special option on CentOS/Alma 7, 8 or 9, or on -Ubuntu 20.04, 22.04 or 24.04, it will automatically try to +By running bits with no special option on CentOS/Alma 7, 8 or 9 (x86-64 or ARM), +or on Ubuntu 20.04, 22.04 or 24.04 (x86-64), it will automatically try to use as many precompiled packages as possible by downloading them from a default central server. By using precompiled packages you lose the ability to pick some of them from your system. If you do not want to use precompiled packages and you @@ -199,16 +212,30 @@ case the incremental recipe will always be executed. While bits does its best to find out which OS / distribution you are using, sometimes it might fail to do so, for example in the case you -start using a new *buntu flavour or a bleeding edge version of Centos. -In order to force the the correct architecture for the build you can use +start using a new *buntu flavour or a bleeding edge version of a distribution. +In order to force the correct architecture for the build you can use the `--architecture` (`-a`) flag with one of the supported options: -- `slc5_x86-64`: Scientific Linux 5 and compatibles, on Intel / AMD x86-64. -- `slc6_x86-64`: Scientific Linux 6 and compatibles, on Intel / AMD x86-64. -- `slc7_x86-64`: CERN Centos 7 and compatibles, on Intel / AMD x86-64. -- `ubuntu1404_x86-64`: Ubuntu 1404 and compatibles, on Intel / AMD x86-64. -- `osx_x86-64`: OSX, on Intel / AMD x86-64. -- `slc7_ppc64`: RHEL7 on POWER8 (LE only for now). +On Linux, x86-64: +- `slc6_x86-64`: RHEL6 / SLC6 compatible +- `slc7_x86-64`: RHEL7 / CC7 compatible +- `slc8_x86-64`: RHEL8 / CC8 compatible +- `slc9_x86-64`: RHEL9 / ALMA9 compatible +- `ubuntu2004_x86-64`: Ubuntu 20.04 compatible +- `ubuntu2204_x86-64`: Ubuntu 22.04 compatible +- `ubuntu2404_x86-64`: Ubuntu 24.04 compatible +- `fedora33_x86-64`: Fedora 33 compatible +- `fedora34_x86-64`: Fedora 34 compatible + +On Linux, ARM: +- `slc9_aarch64`: RHEL9 / ALMA9 compatible + +On Linux, POWER8 / PPC64 (little endian): +- `slc7_ppc64`: RHEL7 / CC7 compatible + +On Mac: +- `osx_x86-64`: Intel +- `osx_arm64`: Apple Silicon ### Running in Docker @@ -234,15 +261,22 @@ option using the same syntax used by Docker. ## Defaults -By default `bits` is tuned to build the production version of ALICE -Offline software, as deployed on the Grid, so some of the choices in -terms of version of the packages and compilation flags are tweaked for -that. For example, ROOT5 is used because that's what is what has been -validated for datataking and the choice will not change until the end of -RUN2 of LHC. In order to change that and use, for example, a more recent -version of ROOT you can use the `--default root6` option which will -enable ROOT6 based builds. For a more complete description of how defaults -works please look at [the reference manual](reference.md#defaults). +By default, `bits` uses the `o2` defaults (`--defaults o2`), which are +optimized for building the ALICE O2 software stack. The defaults system +allows you to specify different sets of build configurations, compiler +flags, and package versions through the `--defaults` option. + +Different defaults can be used to: +- Use different package versions (e.g., different ROOT versions) +- Apply specific compiler flags (e.g., debug builds, optimization levels) +- Enable or disable certain features or packages + +To use a different set of defaults, use the `--defaults ` option, +which will load settings from `CONFIGDIR/defaults-.sh`. For example, +`--defaults o2-epn` would use the `defaults-o2-epn.sh` file. + +For a more complete description of how the defaults system works and how to +create custom defaults, please look at [the reference manual](reference.md#defaults). ## Disabling packages @@ -252,8 +286,8 @@ separated list with the `--disable` option. It is possible to disable packages by adding them to the `disable` keyword of your defaults file (see previous paragraph). See the [defaults-o2.sh](https://github.com/alisw/alidist/blob/master/defaults-o2.sh) -file for an example of how to disable `AliEn-Runtime` and `AliRoot` -when passing `--defaults o2`. +file for an example of how to disable `mesos` and `MySQL` when +passing `--defaults o2`. ## Controlling which system packages are picked up @@ -293,15 +327,12 @@ installation area. ## Upgrading bits -bits is installed via `pip`. In order to upgrade it on most laptops (in -particular Macs) do: +bits can be installed either via `pip`, or by your OS package manager (more info [here](https://alice-doc.github.io/alice-analysis-tutorial/building/custom.html). - pip install --upgrade bits +The way to upgrade it depends on your installation method. If you installed it +via `pip`, you can upgrade it by running: -or in case you need to be root (_e.g._ on Ubuntu and most Linux distributions -for convenience): - - sudo pip install --upgrade bits + pip install --upgrade bits In general updating bits is safe and it should never trigger a rebuild or break compilation of older versions of alidist (i.e. we do try to guarantee @@ -314,10 +345,8 @@ when running the command. You can also upgrade / install a specific version of bits by specifying it on the command line. E.g.: - pip install bits=1.5.5.rc1 + pip install bits=1.17.23 -this is in particular required when you want to try out release candidates (rc) builds which -are masked out by default. ## Rebuilding packages from branches instead of tags @@ -362,6 +391,18 @@ you may want to use: bitsDeps O2 --no-system --outgraph graph.pdf +Additional useful options for `bitsDeps` include: + +- `--neat`: Produce a graph with transitive reduction, removing edges that are + implied by other paths in the graph. This can make complex dependency graphs + easier to read. +- `--outdot FILE`: Keep the intermediate Graphviz dot file in `FILE`. Useful if + you want to manually modify the graph or generate output in different formats. + +For example, to generate a simplified graph and keep the dot file: + + bitsDeps O2 --neat --outdot graph.dot --outgraph graph.pdf + Please run `bitsDeps --help` for further information. ## Using the packages you have built diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..12a7d59a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +requests +pyyaml +distro +jinja2 +# https://github.com/boto/boto3/issues/4398 +boto3<1.36.0 diff --git a/tests/test_build.py b/tests/test_build.py index a1ed6ed6..017bd8fe 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,6 +1,7 @@ from argparse import Namespace import os import os.path +import platform import re import sys import unittest @@ -12,6 +13,19 @@ from bits_helpers.utilities import parseRecipe, resolve_tag from bits_helpers.build import doBuild, storeHashes, generate_initdotsh +# Determine architecture based on platform +def get_test_architecture(): + if sys.platform == 'darwin': + machine = platform.machine() + if machine == 'arm64': + return 'osx_arm64' + else: + return 'osx_x86-64' + else: + return 'slc7_x86-64' + +TEST_ARCHITECTURE = os.environ.get('ARCHITECTURE', get_test_architecture()) + TEST_DEFAULT_RELEASE = """\ package: defaults-release @@ -23,7 +37,7 @@ TEST_ZLIB_RECIPE = """\ package: zlib -version: v1.2.3 +version: v1.3.1 source: https://github.com/madler/zlib tag: master --- @@ -32,7 +46,7 @@ make install """ TEST_ZLIB_GIT_REFS = "8822efa61f2a385e0bc83ca5819d608111b2168a\trefs/heads/master" -TEST_ZLIB_BUILD_HASH = "8cd1f56c450f05ffbba3276bad08eae30f814999" +TEST_ZLIB_BUILD_HASH = "4d6a75f214dc7931a2a7d5ba82ea0568e652cd84" TEST_ROOT_RECIPE = """\ package: ROOT @@ -71,7 +85,7 @@ 87b87c4322d2a3fad315c919cb2e2dd73f2154dc\trefs/heads/master f7b336611753f1f4aaa94222b0d620748ae230c0\trefs/heads/v6-08-00-patches f7b336611753f1f4aaa94222b0d620748ae230c0\trefs/tags/test-tag""" -TEST_ROOT_BUILD_HASH = ("8ec3f41b6b585ef86a02e9c595eed67f34d63f08") +TEST_ROOT_BUILD_HASH = ("1f3c771080f71b6c0d2e3d7a285698a20035da12") TEST_EXTRA_RECIPE = """\ @@ -88,20 +102,20 @@ ba22\trefs/tags/v1 ba22\trefs/tags/v2 baad\trefs/tags/v3""" -TEST_EXTRA_BUILD_HASH = ("5afae57bfc6a374e74c1c4427698ab5edebce0bc") +TEST_EXTRA_BUILD_HASH = ("6e7bc4976abf77b558cf7faf575ec51670f8d0e5") GIT_CLONE_REF_ZLIB_ARGS = ("clone", "--bare", "https://github.com/madler/zlib", "/sw/MIRROR/zlib", "--filter=blob:none"), ".", False GIT_CLONE_SRC_ZLIB_ARGS = ("clone", "-n", "https://github.com/madler/zlib", - "/sw/SOURCES/zlib/v1.2.3/8822efa61f", + "/sw/SOURCES/zlib/v1.3.1/8822efa61f", "--dissociate", "--reference", "/sw/MIRROR/zlib", "--filter=blob:none"), ".", False GIT_SET_URL_ZLIB_ARGS = ("remote", "set-url", "--push", "origin", "https://github.com/madler/zlib"), \ - "/sw/SOURCES/zlib/v1.2.3/8822efa61f", False + "/sw/SOURCES/zlib/v1.3.1/8822efa61f", False GIT_CHECKOUT_ZLIB_ARGS = ("checkout", "-f", "master"), \ - "/sw/SOURCES/zlib/v1.2.3/8822efa61f", False + "/sw/SOURCES/zlib/v1.3.1/8822efa61f", False -GIT_FETCH_REF_ROOT_ARGS = ("fetch", "-f", "--filter=blob:none", "https://github.com/root-mirror/root", "+refs/tags/*:refs/tags/*", +GIT_FETCH_REF_ROOT_ARGS = ("fetch", "-f", "--prune", "--filter=blob:none", "https://github.com/root-mirror/root", "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*"), "/sw/MIRROR/root", False GIT_CLONE_SRC_ROOT_ARGS = ("clone", "-n", "https://github.com/root-mirror/root", "/sw/SOURCES/ROOT/v6-08-30/f7b3366117", @@ -143,9 +157,9 @@ def dummy_open(x, mode="r", encoding=None, errors=None): "/sw/BUILD/%s/defaults-release/.build_succeeded" % TEST_DEFAULT_RELEASE_BUILD_HASH: (0, StringIO("0")), "/sw/BUILD/%s/zlib/.build_succeeded" % TEST_ZLIB_BUILD_HASH: (0, StringIO("0")), "/sw/BUILD/%s/ROOT/.build_succeeded" % TEST_ROOT_BUILD_HASH: (0, StringIO("0")), - "/sw/osx_x86-64/defaults-release/v1-1/.build-hash": (1, StringIO(TEST_DEFAULT_RELEASE_BUILD_HASH)), - "/sw/osx_x86-64/zlib/v1.2.3-local1/.build-hash": (1, StringIO(TEST_ZLIB_BUILD_HASH)), - "/sw/osx_x86-64/ROOT/v6-08-30-local1/.build-hash": (1, StringIO(TEST_ROOT_BUILD_HASH)) + f"/sw/{TEST_ARCHITECTURE}/defaults-release/v1-1/.build-hash": (1, StringIO(TEST_DEFAULT_RELEASE_BUILD_HASH)), + f"/sw/{TEST_ARCHITECTURE}/zlib/v1.3.1-local1/.build-hash": (1, StringIO(TEST_ZLIB_BUILD_HASH)), + f"/sw/{TEST_ARCHITECTURE}/ROOT/v6-08-30-local1/.build-hash": (1, StringIO(TEST_ROOT_BUILD_HASH)) }[x] except KeyError: return DEFAULT @@ -163,17 +177,16 @@ def dummy_execute(x, **kwds): if re.match(".*ln -sfn.*TARS", s): return 0 return { - "/bin/bash -e -x /sw/SPECS/osx_x86-64/defaults-release/v1-1/build.sh 2>&1": 0, - '/bin/bash -e -x /sw/SPECS/osx_x86-64/zlib/v1.2.3-local1/build.sh 2>&1': 0, - '/bin/bash -e -x /sw/SPECS/osx_x86-64/ROOT/v6-08-30-local1/build.sh 2>&1': 0, + f"/bin/bash -e -x /sw/SPECS/{TEST_ARCHITECTURE}/defaults-release/v1-1/build.sh 2>&1": 0, + f'/bin/bash -e -x /sw/SPECS/{TEST_ARCHITECTURE}/zlib/v1.3.1-local1/build.sh 2>&1': 0, + f'/bin/bash -e -x /sw/SPECS/{TEST_ARCHITECTURE}/ROOT/v6-08-30-local1/build.sh 2>&1': 0, }[s] def dummy_readlink(x): return { - "/sw/TARS/osx_x86-64/defaults-release/defaults-release-v1-1.osx_x86-64.tar.gz": - "../../osx_x86-64/store/%s/%s/defaults-release-v1-1.osx_x86-64.tar.gz" % - (TEST_DEFAULT_RELEASE_BUILD_HASH[:2], TEST_DEFAULT_RELEASE_BUILD_HASH) + f"/sw/TARS/{TEST_ARCHITECTURE}/defaults-release/defaults-release-v1-1.{TEST_ARCHITECTURE}.tar.gz": + f"../../{TEST_ARCHITECTURE}/store/{TEST_DEFAULT_RELEASE_BUILD_HASH[:2]}/{TEST_DEFAULT_RELEASE_BUILD_HASH}/defaults-release-v1-1.{TEST_ARCHITECTURE}.tar.gz" }[x] @@ -225,13 +238,11 @@ class BuildTestCase(unittest.TestCase): @patch("os.listdir") @patch("bits_helpers.build.glob", new=lambda pattern: { "*": ["zlib"], - "/sw/TARS/osx_x86-64/store/%s/%s/*gz" % (TEST_DEFAULT_RELEASE_BUILD_HASH[:2], - TEST_DEFAULT_RELEASE_BUILD_HASH): [], - "/sw/TARS/osx_x86-64/store/%s/%s/*gz" % (TEST_ZLIB_BUILD_HASH[:2], TEST_ZLIB_BUILD_HASH): [], - "/sw/TARS/osx_x86-64/store/%s/%s/*gz" % (TEST_ROOT_BUILD_HASH[:2], TEST_ROOT_BUILD_HASH): [], - "/sw/TARS/osx_x86-64/defaults-release/defaults-release-v1-1.osx_x86-64.tar.gz": - ["../../osx_x86-64/store/%s/%s/defaults-release-v1-1.osx_x86-64.tar.gz" % - (TEST_DEFAULT_RELEASE_BUILD_HASH[:2], TEST_DEFAULT_RELEASE_BUILD_HASH)], + f"/sw/TARS/{TEST_ARCHITECTURE}/store/{TEST_DEFAULT_RELEASE_BUILD_HASH[:2]}/{TEST_DEFAULT_RELEASE_BUILD_HASH}/*gz": [], + f"/sw/TARS/{TEST_ARCHITECTURE}/store/{TEST_ZLIB_BUILD_HASH[:2]}/{TEST_ZLIB_BUILD_HASH}/*gz": [], + f"/sw/TARS/{TEST_ARCHITECTURE}/store/{TEST_ROOT_BUILD_HASH[:2]}/{TEST_ROOT_BUILD_HASH}/*gz": [], + f"/sw/TARS/{TEST_ARCHITECTURE}/defaults-release/defaults-release-v1-1.{TEST_ARCHITECTURE}.tar.gz": + [f"../../{TEST_ARCHITECTURE}/store/{TEST_DEFAULT_RELEASE_BUILD_HASH[:2]}/{TEST_DEFAULT_RELEASE_BUILD_HASH}/defaults-release-v1-1.{TEST_ARCHITECTURE}.tar.gz"], }[pattern]) @patch("bits_helpers.build.readlink", new=dummy_readlink) @patch("bits_helpers.build.banner", new=MagicMock(return_value=None)) @@ -244,9 +255,9 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_git_git mock_debug.side_effect = lambda *args: None mock_warning.side_effect = lambda *args: None mock_listdir.side_effect = lambda directory: { - "/sw/TARS/osx_x86-64/defaults-release": ["defaults-release-v1-1.osx_x86-64.tar.gz"], - "/sw/TARS/osx_x86-64/zlib": [], - "/sw/TARS/osx_x86-64/ROOT": [], + f"/sw/TARS/{TEST_ARCHITECTURE}/defaults-release": [f"defaults-release-v1-1.{TEST_ARCHITECTURE}.tar.gz"], + f"/sw/TARS/{TEST_ARCHITECTURE}/zlib": [], + f"/sw/TARS/{TEST_ARCHITECTURE}/ROOT": [], }.get(directory, DEFAULT) os.environ["BITS_NO_ANALYTICS"] = "1" @@ -258,7 +269,7 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_git_git docker=False, dockerImage=None, docker_extra_args=["--network=host"], - architecture="osx_x86-64", + architecture=TEST_ARCHITECTURE, workDir="/sw", pkgname=["root"], configDir="/alidist", @@ -272,7 +283,7 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_git_git debug=True, dryRun=False, aggressiveCleanup=False, - environment={}, + environment=[], autoCleanup=False, noDevel=[], onlyDeps=False, @@ -322,7 +333,7 @@ def mkcall(args): doBuild(args, mock_parser) mock_warning.assert_called_with("%s.sh contains a recipe, which will be ignored", "defaults-release") mock_debug.assert_called_with("Everything done") - mock_listdir.assert_called_with("/sw/TARS/osx_x86-64/ROOT") + mock_listdir.assert_called_with(f"/sw/TARS/{TEST_ARCHITECTURE}/ROOT") # We can't compare directly against the list of calls here as they # might happen in any order. mock_git_git.assert_has_calls(common_calls + [ @@ -409,8 +420,8 @@ def test_initdotsh(self) -> None: self.assertNotIn("Extra", complete_initdotsh) # Dependencies must be loaded both for this build and for subsequent ones. - self.assertIn('. "$WORK_DIR/$BITS_ARCH_PREFIX"/zlib/v1.2.3-1/etc/profile.d/init.sh', setup_initdotsh) - self.assertIn('. "$WORK_DIR/$BITS_ARCH_PREFIX"/zlib/v1.2.3-1/etc/profile.d/init.sh', complete_initdotsh) + self.assertIn('. "$WORK_DIR/$BITS_ARCH_PREFIX"/zlib/v1.3.1-1/etc/profile.d/init.sh', setup_initdotsh) + self.assertIn('. "$WORK_DIR/$BITS_ARCH_PREFIX"/zlib/v1.3.1-1/etc/profile.d/init.sh', complete_initdotsh) # ROOT-specific variables must not be set during ROOT's build yet... self.assertNotIn("export ROOT_VERSION=", setup_initdotsh) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 2934f425..f1f604f4 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -26,7 +26,7 @@ def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput): mock_getoutput.assert_called_with(["docker", "run", "--detach", "--rm", "--entrypoint=", "extra arg", "image", "sleep", "inf"]) getstatusoutput_docker("echo foo") - mock_getstatusoutput.assert_called_with("docker container exec container-id bash -c 'echo foo'", cwd=None) + mock_getstatusoutput.assert_called_with(["docker", "container", "exec", "container-id", "bash", "-c", "echo foo"], cwd=None) mock_getstatusoutput.assert_called_with("docker container kill container-id") mock_getoutput.reset_mock() @@ -38,6 +38,58 @@ def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput): mock_getstatusoutput.reset_mock() mock_getstatusoutput.assert_not_called() + @mock.patch("bits_helpers.cmd.getoutput") + @mock.patch("bits_helpers.cmd.getstatusoutput") + def test_DockerRunner_with_env_vars(self, mock_getstatusoutput, mock_getoutput): + # Test that environment variables are properly injected into docker exec commands. + mock_getoutput.side_effect = lambda cmd: "container-id\n" + + # Test with environment variables + extra_env = {"TEST_VAR": "test_value", "ANOTHER_VAR": "another_value"} + with DockerRunner("image", extra_env=extra_env) as getstatusoutput_docker: + # Verify container creation includes environment variables + mock_getoutput.assert_called_with(["docker", "run", "--detach", + "-e", "TEST_VAR=test_value", + "-e", "ANOTHER_VAR=another_value", + "--rm", "--entrypoint=", "image", "sleep", "inf"]) + + # Test that exec command includes environment variables + getstatusoutput_docker("echo test") + mock_getstatusoutput.assert_called_with(["docker", "container", "exec", + "-e", "TEST_VAR=test_value", + "-e", "ANOTHER_VAR=another_value", + "container-id", "bash", "-c", "echo test"], cwd=None) + + # Test host execution with environment variables + mock_getoutput.reset_mock() + mock_getstatusoutput.reset_mock() + with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker: + mock_getoutput.assert_not_called() + getstatusoutput_docker("echo test") + mock_getstatusoutput.assert_called_with("env TEST_VAR=test_value ANOTHER_VAR=another_value /bin/bash -c 'echo test'", cwd=None) + + @mock.patch("bits_helpers.cmd.getoutput") + @mock.patch("bits_helpers.cmd.getstatusoutput") + def test_DockerRunner_multiline_env_var(self, mock_getstatusoutput, mock_getoutput): + multiline_value = "line1\nline2\nline3" + extra_env = {"MULTILINE_VAR": multiline_value} + + with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker: + mock_getoutput.assert_not_called() + getstatusoutput_docker("echo test") + mock_getstatusoutput.assert_called_with("env MULTILINE_VAR='line1\nline2\nline3' /bin/bash -c 'echo test'", cwd=None) + + @mock.patch("bits_helpers.cmd.getoutput") + @mock.patch("bits_helpers.cmd.getstatusoutput") + def test_DockerRunner_env_var_with_semicolon(self, mock_getstatusoutput, mock_getoutput): + semicolon_value = "value1;value2;value3" + extra_env = {"SEMICOLON_VAR": semicolon_value} + + with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker: + mock_getoutput.assert_not_called() + getstatusoutput_docker("echo test") + mock_getstatusoutput.assert_called_with("env SEMICOLON_VAR='value1;value2;value3' /bin/bash -c 'echo test'", cwd=None) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_deps.py b/tests/test_deps.py index c4922a1f..ec5d77e5 100644 --- a/tests/test_deps.py +++ b/tests/test_deps.py @@ -66,6 +66,11 @@ def depsOpen(fn, mode): outgraph="/tmp/outgraph.pdf", package="AliRoot", defaults="release", + environment=[]) + def fake_exists(n): + return {"/alidist/aliroot.sh": True} + with patch.object(os.path, "exists", fake_exists): + doDeps(args, MagicMock()) xdefaults=None) diff --git a/tests/test_doctor.py b/tests/test_doctor.py index fa239ebf..ea5aa74f 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -91,6 +91,7 @@ def resetOut(): architecture="osx_x86-64", disable=[], defaults="release", + environment=[]) xdefaults=None) # What to call (longer names deprecated in Python 3.5+) diff --git a/tests/test_init.py b/tests/test_init.py index d3e470eb..0522188f 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -48,6 +48,7 @@ def test_doDryRunInit(self, mock_os, mock_path, mock_info) -> None: dryRun = True, fetchRepos = False, architecture = "slc7_x86-64", + environment = {}, xdefaults = None ) self.assertRaises(SystemExit, doInit, args) @@ -88,6 +89,7 @@ def test_doRealInit(self, mock_read_defaults, mock_open, mock_update_reference, dryRun = False, fetchRepos = False, architecture = "slc7_x86-64", + environment = {}, xdefaults = None ) doInit(args) diff --git a/tests/test_sync.py b/tests/test_sync.py index af898261..2fa1d7ac 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -16,19 +16,19 @@ BAD_HASH = "baadf00dbaadf00dbaadf00dbaadf00dbaadf00d" NONEXISTENT_HASH = "TRIGGERS_A_404" GOOD_SPEC = { # fully present on the remote store - "package": PACKAGE, "version": "v1.2.3", "revision": "1", + "package": PACKAGE, "version": "v1.3.1", "revision": "1", "hash": GOOD_HASH, "remote_revision_hash": GOOD_HASH, "remote_hashes": [GOOD_HASH], } BAD_SPEC = { # partially present on the remote store - "package": PACKAGE, "version": "v1.2.3", "revision": "2", + "package": PACKAGE, "version": "v1.3.1", "revision": "2", "hash": BAD_HASH, "remote_revision_hash": BAD_HASH, "remote_hashes": [BAD_HASH], } MISSING_SPEC = { # completely absent from the remote store - "package": PACKAGE, "version": "v1.2.3", "revision": "3", + "package": PACKAGE, "version": "v1.3.1", "revision": "3", "hash": NONEXISTENT_HASH, "remote_revision_hash": NONEXISTENT_HASH, "remote_hashes": [NONEXISTENT_HASH], diff --git a/tests/test_utilities.py b/tests/test_utilities.py index c8d14220..2c781606 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -114,12 +114,102 @@ BUG_REPORT_URL="https://bugs.sabayon.org/" """ +ALMA_8_OS_RELEASE = """ +NAME="AlmaLinux" +VERSION="8.10 (Cerulean Leopard)" +ID="almalinux" +ID_LIKE="rhel centos fedora" +VERSION_ID="8.10" +PLATFORM_ID="platform:el8" +PRETTY_NAME="AlmaLinux 8.10 (Cerulean Leopard)" +ANSI_COLOR="0;34" +LOGO="fedora-logo-icon" +CPE_NAME="cpe:/o:almalinux:almalinux:8::baseos" +HOME_URL="https://almalinux.org/" +DOCUMENTATION_URL="https://wiki.almalinux.org/" +BUG_REPORT_URL="https://bugs.almalinux.org/" + +ALMALINUX_MANTISBT_PROJECT="AlmaLinux-8" +ALMALINUX_MANTISBT_PROJECT_VERSION="8.10" +REDHAT_SUPPORT_PRODUCT="AlmaLinux" +REDHAT_SUPPORT_PRODUCT_VERSION="8.10" +SUPPORT_END=2029-06-01 +""" + +ALMA_9_OS_RELEASE = """ +NAME="AlmaLinux" +VERSION="9.6 (Sage Margay)" +ID="almalinux" +ID_LIKE="rhel centos fedora" +VERSION_ID="9.6" +PLATFORM_ID="platform:el9" +PRETTY_NAME="AlmaLinux 9.6 (Sage Margay)" +ANSI_COLOR="0;34" +LOGO="fedora-logo-icon" +CPE_NAME="cpe:/o:almalinux:almalinux:9::baseos" +HOME_URL="https://almalinux.org/" +DOCUMENTATION_URL="https://wiki.almalinux.org/" +BUG_REPORT_URL="https://bugs.almalinux.org/" + +ALMALINUX_MANTISBT_PROJECT="AlmaLinux-9" +ALMALINUX_MANTISBT_PROJECT_VERSION="9.6" +REDHAT_SUPPORT_PRODUCT="AlmaLinux" +REDHAT_SUPPORT_PRODUCT_VERSION="9.6" +SUPPORT_END=2032-06-01 +""" + +ROCKY_8_OS_RELEASE = """ +NAME="Rocky Linux" +VERSION="8.10 (Green Obsidian)" +ID="rocky" +ID_LIKE="rhel centos fedora" +VERSION_ID="8.10" +PLATFORM_ID="platform:el8" +PRETTY_NAME="Rocky Linux 8.10 (Green Obsidian)" +ANSI_COLOR="0;32" +LOGO="fedora-logo-icon" +CPE_NAME="cpe:/o:rocky:rocky:8:GA" +HOME_URL="https://rockylinux.org/" +BUG_REPORT_URL="https://bugs.rockylinux.org/" +SUPPORT_END="2029-05-31" +ROCKY_SUPPORT_PRODUCT="Rocky-Linux-8" +ROCKY_SUPPORT_PRODUCT_VERSION="8.10" +REDHAT_SUPPORT_PRODUCT="Rocky Linux" +REDHAT_SUPPORT_PRODUCT_VERSION="8.10" +""" + +ROCKY_9_OS_RELEASE = """ +NAME="Rocky Linux" +VERSION="9.6 (Blue Onyx)" +ID="rocky" +ID_LIKE="rhel centos fedora" +VERSION_ID="9.6" +PLATFORM_ID="platform:el9" +PRETTY_NAME="Rocky Linux 9.6 (Blue Onyx)" +ANSI_COLOR="0;32" +LOGO="fedora-logo-icon" +CPE_NAME="cpe:/o:rocky:rocky:9::baseos" +HOME_URL="https://rockylinux.org/" +VENDOR_NAME="RESF" +VENDOR_URL="https://resf.org/" +BUG_REPORT_URL="https://bugs.rockylinux.org/" +SUPPORT_END="2032-05-31" +ROCKY_SUPPORT_PRODUCT="Rocky-Linux-9" +ROCKY_SUPPORT_PRODUCT_VERSION="9.6" +REDHAT_SUPPORT_PRODUCT="Rocky Linux" +REDHAT_SUPPORT_PRODUCT_VERSION="9.6" +""" + architecturePayloads = [ ['osx_x86-64', False, [], ('','',''), 'Darwin', 'x86-64'], ['osx_arm64', False, [], ('','',''), 'Darwin', 'arm64'], ['slc5_x86-64', False, [], ('redhat', '5.XX', 'Boron'), 'Linux', 'x86-64'], ['slc6_x86-64', False, [], ('centos', '6.X', 'Carbon'), 'Linux', 'x86-64'], ['slc7_x86-64', False, [], ('centos', '7.X', 'Ptor'), 'Linux', 'x86-64'], + ['slc8_x86-64', True, ALMA_8_OS_RELEASE.split("\n"), ('AlmaLinux', '8.10', 'Cerulean Leopard'), 'Linux', 'x86_64'], + ['slc8_x86-64', True, ROCKY_8_OS_RELEASE.split("\n"), ('Rocky Linux', '8.10', 'Green Obsidian'), 'Linux', 'x86_64'], + ['slc9_x86-64', True, ALMA_9_OS_RELEASE.split("\n"), ('AlmaLinux', '9.6', 'Sage Margay'), 'Linux', 'x86_64'], + ['slc9_x86-64', True, ROCKY_9_OS_RELEASE.split("\n"), ('Rocky Linux', '9.6', 'Blue Onyx'), 'Linux', 'x86_64'], ['ubuntu1804_x86-64', True, UBUNTU_1804_OS_RELEASE.split("\n"), ('Ubuntu', '18.04', 'bionic'), 'Linux', 'x86-64'], ['ubuntu1604_x86-64', True, UBUNTU_1604_OS_RELEASE.split("\n"), ('Ubuntu', '16.04', 'xenial'), 'Linux', 'x86-64'], ['ubuntu1510_x86-64', False, [], ('Ubuntu', '15.10', 'wily'), 'Linux', 'x86-64'], diff --git a/tests/test_workarea.py b/tests/test_workarea.py index a16c6b58..7dc2eb30 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -57,7 +57,7 @@ def test_reference_sources_updated(self, mock_git, mock_open, mock_makedirs, moc mock_exists.assert_has_calls([]) mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd(), exist_ok=True) mock_git.assert_called_once_with([ - "fetch", "-f", "--filter=blob:none", spec["source"], "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*", + "fetch", "-f", "--prune", "--filter=blob:none", spec["source"], "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*", ], directory="%s/sw/MIRROR/aliroot" % getcwd(), check=False, prompt=True) self.assertEqual(spec.get("reference"), "%s/sw/MIRROR/aliroot" % getcwd()) diff --git a/tox.ini b/tox.ini index ed18b650..fbeef159 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.20 -envlist = lint, py{37, 38, 39, 310, 311, 312, 313}, darwin +envlist = lint, py{37, 38, 39, 310, 311, 312, 313, 314}, darwin [gh-actions] # The "lint" job is run separately. @@ -12,6 +12,7 @@ python = 3.11: py311 3.12: py312 3.13: py313 + 3.14: py314 ; [testenv:lint] ; # Warning: This environment inherits settings from the main [testenv] section. @@ -121,7 +122,7 @@ commands = darwin: sl clone https://github.com/alisw/alidist alidist-sapling darwin: coverage run --source={toxinidir} -a {toxinidir}/bitsBuild -a {env:ARCHITECTURE} -c alidist-sapling --no-system --disable GCC-Toolchain build zlib darwin: rm -fr zlib - darwin: sl clone https://github.com/alisw/zlib + darwin: sl clone https://github.com/madler/zlib darwin: coverage run --source={toxinidir} -a {toxinidir}/bitsBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib touch zlib/foo darwin: coverage run --source={toxinidir} -a {toxinidir}/bitsBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib @@ -139,3 +140,11 @@ exclude_lines = raise NotImplementedError # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: + +[testenv:check-readme] +# Check the README.rst file for common issues. +# The pypa publishing job fails if this fails. +deps = + rstcheck +commands = + rstcheck {toxinidir}/README.rst