Several aliBuild ports

.github/workflows/pr-check.yml

@@ -31,6 +31,7 @@ jobs:
           - '3.11'
           - '3.12'
           - '3.13'
+          - '3.14'
 
     steps:
       - uses: actions/checkout@v4
@@ -78,6 +79,7 @@ jobs:
           - '3.11'
           - '3.12'
           - '3.13'
+          - '3.14'
 
     steps:
       - uses: actions/checkout@v4
@@ -134,3 +136,22 @@ jobs:
   #
   #     - name: Run linters
   #       run: tox -e lint
+
+  check-readme:
+    name: Check README
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+
+      - name: Install dependencies
+        run: python -m pip install --upgrade tox tox-gh-actions
+
+      - name: Run README check
+        run: tox -e check-readme
+

.github/workflows/release.yml

@@ -1,30 +1,32 @@
 ---
 name: Release
 
-'on':
+"on":
   release:
     types: [published]
 
 permissions:
-  contents: read  # required for github.ref to be set
+  contents: read # required for github.ref to be set
 
 jobs:
   pypi-publish:
     name: Upload release to PyPI
     runs-on: ubuntu-latest
     environment:
-      name: pypi
-      url: https://pypi.org/p/alibuild
+      name: release
+      url: https://pypi.org/p/bits # TODO: update with actual package name
     permissions:
-      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
     steps:
-    # retrieve your distributions here
-    - name: Install build dependencies
-      run: python -m pip install --upgrade setuptools build pip
-    - name: Build the Python distribution
-      run: python -m build
-    - name: Publish package distributions to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install build dependencies
+        run: python -m pip install --upgrade setuptools build pip
+      - name: Build the Python distribution
+        run: python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
 
   brew:
     name: Update Homebrew formula
@@ -39,15 +41,14 @@ jobs:
       - name: Update bits formula with new version
         run: |
           set -x
-          json=$(curl -fSsL "https://pypi.org/pypi/bits/${GITHUB_REF#refs/tags/}/json")
-          sha=$(echo "$json" | jq -r '.urls[0].digests.sha256')
-          url=$(echo "$json" | jq -r '.urls[0].url')
+          json=$(curl -fSsL "https://pypi.org/pypi/bits/${GITHUB_REF#refs/tags/}/json") # TODO: update with actual package name
+          sha=$(echo "$json" | jq -r '.urls.[] | select(.filename | endswith(".tar.gz")) | .digests.sha256')
+          url=$(echo "$json" | jq -r '.urls.[] | select(.filename | endswith(".tar.gz")) | .url')
           # Replace keys with two leading spaces only, so we get the toplevel
           # ones applying to bits, not those for dependencies.
           sed -i.bak "
           s|^  url .*|  url \"$url\"|;
           s|^  sha256 .*|  sha256 \"$sha\"|;
-          s|^  version .*|  version \"${GITHUB_REF#refs/tags/v}\"|
           " "$(brew formula bits)"
           rm -f "$(brew formula bits).bak"
       - name: Regenerate bits formula
@@ -70,6 +71,6 @@ jobs:
           git add bits.rb
           git commit -m "Update bits to ${GITHUB_REF#refs/tags/}"
           git show
-          git push "https://bits:$GITHUB_TOKEN@github.com/bitsorg/homebrew-system-deps" master
+          git push "https://bits:$GITHUB_TOKEN@github.com/bits/homebrew-system-deps" master # TODO: update user and repo
         env:
           GITHUB_TOKEN: ${{ secrets.homebrew_repo_token }}

README.rst

@@ -35,14 +35,22 @@ Pre-requisites
 If you are using bits directly from git clone, you should make sure
 you have the dependencies installed. The easiest way to do this is to run::
 
+    # Optional, make a venv so the dependencies are not installed globally
+    python -m venv .venv
+    source .venv/bin/activate
     pip install -e .
 
 
-For developers
-==========
+Contributing
+============
+
 
 If you want to contribute to bits, you can run the tests with::
 
+    # Optional, make a venv so the dependencies are not installed globally
+    python -m venv .venv
+    source .venv/bin/activate
+
     pip install -e .[test] # Only needed once
     tox
 
@@ -56,6 +64,13 @@ You can also run only the unit tests (it's a lot faster than the full suite) wit
 
 To run the documentation locally, you can use::
 
+    # Optional, make a venv so the dependencies are not installed globally
+    python -m venv .venv
+    source .venv/bin/activate
+
+    # Install dependencies for the docs, check pyproject.toml for more info
     pip install -e .[docs]
+
+    # Run the docs
     cd docs
     mkdocs serve

bits_helpers/args.py

@@ -262,6 +262,8 @@ def doParseArgs():
                              help=("Assume we're not building %(metavar)s and all its (unique) dependencies. "
                                    "You can specify this option multiple times or separate multiple arguments "
                                    "with commas."))
+  doctor_parser.add_argument("-e", dest="environment", action="append", default=[],
+                            help="KEY=VALUE binding to add to the build environment. May be specified multiple times.")
 
   doctor_system = doctor_parser.add_mutually_exclusive_group()
   doctor_system.add_argument("--always-prefer-system", dest="preferSystem", action="store_true",

bits_helpers/build.py

@@ -60,7 +60,7 @@ def update_repo(package, git_prompt):
         specs[package]["scm"] = Git()
         if specs[package]["is_devel_pkg"]:
             specs[package]["source"] = os.path.join(os.getcwd(), specs[package]["package"])
-            if exists(os.path.join(specs[package]["source"], ".sl")):
+            if exists(os.path.join(specs[package]["source"], ".sl")) or exists(os.path.join(specs[package]["source"], ".git/sl")):
                 specs[package]["scm"] = Sapling()
         updateReferenceRepoSpec(args.referenceSources, package, specs[package],
                                 fetch=args.fetchRepos, allowGitPrompt=git_prompt)
@@ -515,7 +515,10 @@ def doBuild(args, parser):
 
   install_wrapper_script("git", workDir)
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker:
+  extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
+  extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
+
+  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     def performPreferCheckWithTempDir(pkg, cmd):
       with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir:
         return getstatusoutput_docker(cmd, cwd=temp_dir)
@@ -1150,12 +1153,13 @@ def performPreferCheckWithTempDir(pkg, cmd):
     if args.docker:
       build_command = (
         "docker run --rm --entrypoint= --user $(id -u):$(id -g) "
-        "-v {workdir}:/sw -v {scriptDir}/build.sh:/build.sh:ro "
+        "-v {workdir}:/sw -v{configDir}:/alidist:ro -v {scriptDir}/build.sh:/build.sh:ro "
         "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} "
-        "-e WORK_DIR_OVERRIDE=/sw {extraArgs} {image} bash -ex /build.sh"
+        "-e WORK_DIR_OVERRIDE=/sw -e BITS_CONFIG_DIR_OVERRIDE=/alidist {extraArgs} {image} bash -ex /build.sh"
       ).format(
         image=quote(args.dockerImage),
         workdir=quote(abspath(args.workDir)),
+        configDir=quote(abspath(args.configDir)),
         scriptDir=quote(scriptDir),
         extraArgs=" ".join(map(quote, args.docker_extra_args)),
         additionalEnv=" ".join(

bits_helpers/build_template.sh

@@ -16,6 +16,7 @@ cpath=$(which gcc)
 function hash() { true; }
 
 export WORK_DIR="${WORK_DIR_OVERRIDE:-%(workDir)s}"
+export BITS_CONFIG_DIR="${BITS_CONFIG_DIR_OVERRIDE:-%(configDir)s}"
 
 # Insert our own wrapper scripts into $PATH, patched to use the system OpenSSL,
 # instead of the one we build ourselves.
@@ -108,9 +109,9 @@ unset DYLD_LIBRARY_PATH
 EOF
 
 cd "$BUILDROOT"
-ln -snf $PKGHASH "${BUILDROOT}-latest"
+ln -snf "$PKGHASH" "$ALIBUILD_BUILD_WORK_DIR/BUILD/$PKGNAME-latest"
 if [[ $DEVEL_PREFIX ]]; then
-  ln -snf $PKGHASH "${BUILDROOT}-latest-$DEVEL_PREFIX"
+  ln -snf "$PKGHASH" "$ALIBUILD_BUILD_WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX"
 fi
 
 cd "$BUILDDIR"

bits_helpers/cmd.py

@@ -78,25 +78,42 @@ class DockerRunner:
   instead.
   """
 
-  def __init__(self, docker_image, docker_run_args=()) -> None:
+  def __init__(self, docker_image, docker_run_args=(), extra_env={}, extra_volumes=[]) -> None:
     self._docker_image = docker_image
     self._docker_run_args = docker_run_args
     self._container = None
+    self._extra_env = extra_env
+    self._extra_volumes = extra_volumes
 
   def __enter__(self):
     if self._docker_image:
       # "sleep inf" pauses forever, until we kill it.
-      cmd = ["docker", "run", "--detach", "--rm", "--entrypoint="]
+      envOpts = []
+      volumes = []
+      for env in self._extra_env.items():
+        envOpts.append("-e")
+        envOpts.append(f"{env[0]}={env[1]}")
+      for v in self._extra_volumes:
+        volumes.append("-v")
+        volumes.append(v)
+      cmd = ["docker", "run", "--detach"] + envOpts + volumes + ["--rm", "--entrypoint="]
       cmd += self._docker_run_args
       cmd += [self._docker_image, "sleep", "inf"]
       self._container = getoutput(cmd).strip()
 
     def getstatusoutput_docker(cmd, cwd=None):
       if self._container is None:
-        return getstatusoutput("{} -c {}".format(BASH, quote(cmd)), cwd=cwd)
-      return getstatusoutput("docker container exec {} bash -c {}"
-                             .format(quote(self._container), quote(cmd)),
-                             cwd=cwd)
+        command_prefix=""
+        if self._extra_env:
+          command_prefix="env " + " ".join("{}={}".format(k, quote(v)) for (k,v) in self._extra_env.items()) + " "
+        return getstatusoutput("{}{} -c {}".format(command_prefix, BASH, quote(cmd))
+                             , cwd=cwd)
+      envOpts = []
+      for env in self._extra_env.items():
+        envOpts.append("-e")
+        envOpts.append("{}={}".format(env[0], env[1]))
+      exec_cmd = ["docker", "container", "exec"] + envOpts + [self._container, "bash", "-c", cmd]
+      return getstatusoutput(exec_cmd, cwd=cwd)
 
     return getstatusoutput_docker
 

bits_helpers/deps.py

@@ -4,7 +4,7 @@
 from bits_helpers.utilities import parseDefaults, readDefaults, getPackageList, validateDefaults
 from bits_helpers.cmd import DockerRunner, execute
 from tempfile import NamedTemporaryFile
-from os import remove
+from os import remove, path
 
 def doDeps(args, parser):
 
@@ -16,7 +16,14 @@ def doDeps(args, parser):
   specs = {}
   defaultsReader = lambda: readDefaults(args.configDir, args.defaults, parser.error, args.architecture, args.xdefaults)
   (err, overrides, taps) = parseDefaults(args.disable, defaultsReader, debug)
-  with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker:
+
+  extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else path.abspath(args.configDir)}
+  extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
+
+  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
+    def performCheck(pkg, cmd):
+      return getstatusoutput_docker(cmd)
+
     systemPackages, ownPackages, failed, validDefaults = \
       getPackageList(packages                = [args.package],
                      specs                   = specs,
@@ -26,8 +33,8 @@ def doDeps(args, parser):
                      architecture            = args.architecture,
                      disable                 = args.disable,
                      defaults                = args.defaults,
-                     performPreferCheck      = lambda pkg, cmd: getstatusoutput_docker(cmd),
-                     performRequirementCheck = lambda pkg, cmd: getstatusoutput_docker(cmd),
+                     performPreferCheck      = performCheck,
+                     performRequirementCheck = performCheck,
                      performValidateDefaults = lambda spec: validateDefaults(spec, args.defaults),
                      overrides               = overrides,
                      taps                    = taps,

bits_helpers/doctor.py

@@ -22,7 +22,8 @@ def checkPreferSystem(spec, cmd, homebrew_replacement, getstatusoutput_docker):
       debug("Package %s can only be managed via bits.", spec["package"])
       return (1, "")
     cmd = homebrew_replacement + cmd
-    err, out = getstatusoutput_docker(cmd)
+    with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{spec['package']}_") as temp_dir:
+        err, out = getstatusoutput_docker(cmd, cwd=temp_dir)
     if not err:
       success("Package %s will be picked up from the system.", spec["package"])
       for x in out.split("\n"):
@@ -40,7 +41,8 @@ def checkRequirements(spec, cmd, homebrew_replacement, getstatusoutput_docker):
       debug("Package %s is not a system requirement.", spec["package"])
       return (0, "")
     cmd = homebrew_replacement + cmd
-    err, out = getstatusoutput_docker(cmd)
+    with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{spec['package']}_") as temp_dir:
+        err, out = getstatusoutput_docker(cmd, cwd=temp_dir)
     if not err:
       success("Required package %s will be picked up from the system.", spec["package"])
       debug("%s", cmd)
@@ -83,7 +85,11 @@ def doDoctor(args, parser):
   # Decide if we can use homebrew. If not, we replace it with "true" so
   # that we do not get spurious messages on linux
   homebrew_replacement = ""
-  with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker:
+
+  extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
+  extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
+
+  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     err, output = getstatusoutput_docker("type c++")
   if err:
     warning("Unable to find system compiler.\n"
@@ -140,10 +146,10 @@ def performValidateDefaults(spec):
       error("%s", msg)
     return (ok,msg,valid)
 
-  with DockerRunner(args.dockerImage, args.docker_extra_args) as getstatusoutput_docker:
-    def performPreferCheckWithTempDir(pkg, cmd):
-      with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir:
-        return getstatusoutput_docker(cmd, cwd=temp_dir)
+  extra_env = {"BITS_CONFIG_DIR": "/alidist" if args.docker else os.path.abspath(args.configDir)}
+  extra_env.update(dict([e.partition('=')[::2] for e in args.environment]))
+
+  with DockerRunner(args.dockerImage, args.docker_extra_args, extra_env=extra_env, extra_volumes=[f"{os.path.abspath(args.configDir)}:/alidist:ro"] if args.docker else []) as getstatusoutput_docker:
     fromSystem, own, failed, validDefaults = \
       getPackageList(packages                = packages,
                      specs                   = specs,
@@ -153,8 +159,8 @@ def performPreferCheckWithTempDir(pkg, cmd):
                      architecture            = args.architecture,
                      disable                 = args.disable,
                      defaults                = args.defaults,
-                     performPreferCheck      = performPreferCheckWithTempDir,
-                     performRequirementCheck = performPreferCheckWithTempDir,
+                     performPreferCheck      = lambda pkg, cmd: checkPreferSystem(pkg, cmd, homebrew_replacement, getstatusoutput_docker),
+                     performRequirementCheck = lambda pkg, cmd: checkRequirements(pkg, cmd, homebrew_replacement, getstatusoutput_docker),
                      performValidateDefaults = performValidateDefaults,
                      overrides               = overrides,
                      taps                    = taps,

bits_helpers/git.py

@@ -68,7 +68,7 @@ def checkoutCmd(self, tag):
     return ["checkout", "-f", tag]
 
   def fetchCmd(self, remote, *refs):
-    return ["fetch", "-f"] + clone_speedup_options() + [remote, *refs]
+    return ["fetch", "-f", "--prune"] + clone_speedup_options() + [remote, *refs]
 
   def setWriteUrlCmd(self, url):
     return ["remote", "set-url", "--push", "origin", url]

bits_helpers/init.py

@@ -3,6 +3,7 @@
 from bits_helpers.log import debug, error, warning, banner, info
 from bits_helpers.log import dieOnError
 from bits_helpers.workarea import updateReferenceRepoSpec
+from bits_helpers.cmd import getstatusoutput
 
 from os.path import join
 import os.path as path
@@ -54,7 +55,7 @@ def doInit(args):
                                          architecture="",
                                          disable=[],
                                          defaults=args.defaults,
-                                         performPreferCheck=lambda *x, **y: (1, ""),
+                                         performPreferCheck=lambda pkg, cmd: getstatusoutput(["bash", "-c", cmd]),
                                          performRequirementCheck=lambda *x, **y: (0, ""),
                                          performValidateDefaults=lambda spec : validateDefaults(spec, args.defaults),
                                          overrides=overrides,