From e320f53a5fafa212e4950d785f5e84b88e271b96 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Fri, 23 May 2025 10:42:33 +0200 Subject: [PATCH] Allow building in a specified directory (#924) This is for the case where workdir is on a slow filesystem, e.g. NFS. Notice we keep devel packages in the old (possibly slow) place, so that we can do incremental builds (in case the "fast" path has some garbage collection like /tmp). --- bits_helpers/build.py | 72 +++++++++++++++++++--------------- bits_helpers/build_template.sh | 11 ++++-- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index cce949fa..6d4a4521 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -898,8 +898,18 @@ def doBuild(args, parser): spec["hash"] = spec["local_revision_hash"] else: spec["hash"] = spec["remote_revision_hash"] + + # We do not use the override for devel packages, because we + # want to avoid having to rebuild things when the /tmp gets cleaned. + if spec["is_devel_pkg"]: + buildWorkDir = args.workDir + else: + buildWorkDir = os.environ.get("BITS_BUILD_WORK_DIR", args.workDir) + + buildRoot = join(buildWorkDir, "BUILD", spec["hash"]) + spec["old_devel_hash"] = readHashFile(join( - workDir, "BUILD", spec["hash"], spec["package"], ".build_succeeded")) + buildRoot, spec["package"], ".build_succeeded")) # Recreate symlinks to this development package builds. if spec["is_devel_pkg"]: @@ -907,9 +917,9 @@ def doBuild(args, parser): # Ignore errors here, because the path we're linking to might not exist # (if this is the first run through the loop). On the second run # through, the path should have been created by the build process. - call_ignoring_oserrors(symlink, spec["hash"], join(workDir, "BUILD", spec["package"] + "-latest")) + call_ignoring_oserrors(symlink, spec["hash"], join(buildWorkDir, "BUILD", spec["package"] + "-latest")) if develPrefix: - call_ignoring_oserrors(symlink, spec["hash"], join(workDir, "BUILD", spec["package"] + "-latest-" + develPrefix)) + call_ignoring_oserrors(symlink, spec["hash"], join(buildWorkDir, "BUILD", spec["package"] + "-latest-" + develPrefix)) # Last package built gets a "latest" mark. call_ignoring_oserrors(symlink, "{version}-{revision}".format(**spec), join(workDir, args.architecture, spec["package"], "latest")) @@ -961,7 +971,7 @@ def doBuild(args, parser): # assuming the package is not a development one. We also can # delete the SOURCES in case we have aggressive-cleanup enabled. if not spec["is_devel_pkg"] and args.autoCleanup: - cleanupDirs = [join(workDir, "BUILD", spec["hash"]), + cleanupDirs = [buildRoot, join(workDir, "INSTALLROOT", spec["hash"])] if args.aggressiveCleanup: cleanupDirs.append(join(workDir, "SOURCES", spec["package"])) @@ -970,13 +980,13 @@ def doBuild(args, parser): for d in cleanupDirs: shutil.rmtree(d.encode("utf8"), True) try: - unlink(join(workDir, "BUILD", spec["package"] + "-latest")) + unlink(join(buildWorkDir, "BUILD", spec["package"] + "-latest")) if "develPrefix" in args: - unlink(join(workDir, "BUILD", spec["package"] + "-latest-" + args.develPrefix)) + unlink(join(buildWorkDir, "BUILD", spec["package"] + "-latest-" + args.develPrefix)) except: pass try: - rmdir(join(workDir, "BUILD")) + rmdir(join(buildWorkDir, "BUILD")) rmdir(join(workDir, "INSTALLROOT")) except: pass @@ -1176,28 +1186,28 @@ def doBuild(args, parser): if spec["is_devel_pkg"]: updatablePkgs.append(spec["package"]) - buildErrMsg = dedent("""\ - Error while executing {sd}/build.sh on `{h}'. - Log can be found in {w}/BUILD/{p}-latest{devSuffix}/log - Please upload it to CERNBox/Dropbox if you intend to request support. - Build directory is {w}/BUILD/{p}-latest{devSuffix}/{p}. - """).format( - h=socket.gethostname(), - sd=scriptDir, - w=abspath(args.workDir), - p=spec["package"], - devSuffix="-" + args.develPrefix - if "develPrefix" in args and spec["is_devel_pkg"] - else "", - ) - if updatablePkgs: - buildErrMsg += dedent(""" - Note that you have packages in development mode. - Devel sources are not updated automatically, you must do it by hand.\n - This problem might be due to one or more outdated devel sources. - To update all development packages required for this build it is usually sufficient to do: - """) - buildErrMsg += "".join("\n ( cd %s && git pull --rebase )" % dp for dp in updatablePkgs) + buildErrMsg = dedent("""\ + Error while executing {sd}/build.sh on `{h}'. + Log can be found in {w}/BUILD/{p}-latest{devSuffix}/log + Please upload it to CERNBox/Dropbox if you intend to request support. + Build directory is {w}/BUILD/{p}-latest{devSuffix}/{p}. + """).format( + h=socket.gethostname(), + sd=scriptDir, + w=buildWorkDir, + p=spec["package"], + devSuffix="-" + args.develPrefix + if "develPrefix" in args and spec["is_devel_pkg"] + else "", + ) + if updatablePkgs: + buildErrMsg += dedent(""" + Note that you have packages in development mode. + Devel sources are not updated automatically, you must do it by hand.\n + This problem might be due to one or more outdated devel sources. + To update all development packages required for this build it is usually sufficient to do: + """) + buildErrMsg += "".join("\n ( cd %s && git pull --rebase )" % dp for dp in updatablePkgs) # Gather build info for the error message try: @@ -1211,7 +1221,7 @@ def doBuild(args, parser): buildErrMsg += dedent(f""" Build info: OS: {detected_arch} - Using aliBuild from alibuild@{__version__ or "unknown"} recipes in alidist@{os.environ["ALIBUILD_ALIDIST_HASH"][:10]} + Using bits from bits@{__version__ or "unknown"} recipes in dist@{os.environ["BITS_DIST_HASH"][:10]} Build arguments: {args_str} """) @@ -1252,7 +1262,7 @@ def doBuild(args, parser): for spec in specs.values(): if spec["is_devel_pkg"]: banner("Build directory for devel package %s:\n%s/BUILD/%s-latest%s/%s", - spec["package"], abspath(args.workDir), spec["package"], + spec["package"], abspath(buildWorkDir), spec["package"], ("-" + args.develPrefix) if "develPrefix" in args else "", spec["package"]) if untrackedFilesDirectories: diff --git a/bits_helpers/build_template.sh b/bits_helpers/build_template.sh index 55597f1a..37a60e35 100644 --- a/bits_helpers/build_template.sh +++ b/bits_helpers/build_template.sh @@ -44,16 +44,19 @@ export PKG_BUILDNUM="$PKGREVISION" export PKGPATH=${ARCHITECTURE}/${PKGNAME}/${PKGVERSION}-${PKGREVISION} mkdir -p "$WORK_DIR/BUILD" "$WORK_DIR/SOURCES" "$WORK_DIR/TARS" \ "$WORK_DIR/SPECS" "$WORK_DIR/INSTALLROOT" -export BUILDROOT="$WORK_DIR/BUILD/$PKGHASH" - # If we are in development mode, then install directly in $WORK_DIR/$PKGPATH, # so that we can do "make install" directly into BUILD/$PKGPATH and have # changes being propagated. +# Moreover, devel packages should always go in the official WORK_DIR if [ -n "$DEVEL_HASH" ]; then + export BITS_BUILD_WORK_DIR="${WORK_DIR}" export INSTALLROOT="$WORK_DIR/$PKGPATH" else export INSTALLROOT="$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH" + export BITS_BUILD_WORK_DIR="${BITS_BUILD_WORK_DIR:-$WORK_DIR}" fi + +export BUILDROOT="$BITS_BUILD_WORK_DIR/BUILD/$PKGHASH" export SOURCEDIR="$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/$COMMIT_HASH" export BUILDDIR="$BUILDROOT/$PKGNAME" @@ -100,9 +103,9 @@ unset DYLD_LIBRARY_PATH EOF cd "$BUILDROOT" -ln -snf $PKGHASH $WORK_DIR/BUILD/$PKGNAME-latest +ln -snf $PKGHASH "${BUILDROOT}-latest" if [[ $DEVEL_PREFIX ]]; then - ln -snf $PKGHASH $WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX + ln -snf $PKGHASH "${BUILDROOT}-latest-$DEVEL_PREFIX" fi cd "$BUILDDIR"