Offload macOS .app assembly off the build->test critical path

pipelines/main/platforms/build_macos.arches

@@ -1,5 +1,5 @@
 # OS       TRIPLET                  ARCH          MAKE_FLAGS                                TIMEOUT
-macos      x86_64-apple-darwin      x86_64        JL_CFLAGS=-Werror,JL_CXXFLAGS=-Werror     .
+macos      x86_64-apple-darwin      x86_64        JL_CFLAGS=-Werror,JL_CXXFLAGS=-Werror     90
 macos      aarch64-apple-darwin     aarch64       JL_CFLAGS=-Werror,JL_CXXFLAGS=-Werror     .
 
 # These special lines allow us to embed default values for the columns above.

pipelines/main/platforms/stage_macos_app.arches

@@ -0,0 +1,10 @@
+# OS       TRIPLET                 TIMEOUT
+macos      x86_64-apple-darwin     .
+macos      aarch64-apple-darwin    .
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# This job only downloads a tree tarball and repackages it into a .app; it does
+# no compilation, so a short timeout is plenty.
+#default TIMEOUT 30

pipelines/main/platforms/stage_macos_app.yml

@@ -0,0 +1,46 @@
+steps:
+  - group: "${GROUP?}"
+    steps:
+      - label: ":linux::macos: stage app ${TRIPLET?}"
+        key: "stage_app_${TRIPLET?}"
+        # julia-pr ONLY. In julia-ci the .app is assembled + signed in the
+        # trusted publish step (see upload_julia.sh), so this off-critical-path
+        # builder would be redundant there. Mirrors the slug guard the publish
+        # trigger uses at the end of the rendered launch pipeline.
+        if: pipeline.slug == "julia-pr"
+        # Depend only on the macOS build (which stages the tree tarball), NOT on
+        # the tests -- so this runs in parallel with the tests and never delays
+        # the build->test critical path.
+        depends_on:
+          - "build_${TRIPLET?}"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: "./.buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/sandbox#v2:
+              # The .app is repackaged from the downloaded macOS tree on a Linux
+              # host (needs make/sed/tar/git/aws), so run in the standard
+              # package_linux rootfs. Keep pinned in sync with the x86_64 row of
+              # build_linux.arches.
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz
+              rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # `git describe` (contrib/mac/app's Makefile) resolves against
+                # the object cache, same as the build_linux step.
+                - "/cache/repos:/cache/repos"
+        timeout_in_minutes: ${TIMEOUT?}
+        soft_fail: ${ALLOW_FAIL?}
+        commands: "bash .buildkite/utilities/stage_macos_app.sh"
+        agents:
+          # Always a Linux x86_64 host regardless of the macOS TRIPLET: repacking
+          # a tree into a .app runs no target binaries, so it is host-agnostic.
+          queue: "build"
+          os: "linux"
+          arch: "x86_64"
+        env:
+          # TRIPLET selects which macOS tarball to download + which staging path
+          # to write; the agent itself is Linux.
+          TRIPLET: "${TRIPLET?}"
+          BUILDKITE_CANCEL_SIGNAL: "SIGQUIT"

utilities/build_julia.sh

@@ -132,20 +132,10 @@ source .buildkite/utilities/upload_to_s3.sh
 # The staging buckets disable object ACLs (public read via bucket policy)
 UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.tar.gz" "${STAGING_TARGET}.tar.gz"
 
-# macOS: assemble the Julia.app here and stage it too. The build runs on a Mac,
-# so contrib/mac/app's tooling (osacompile, etc.) is available; the trusted
-# publish step then only has to codesign + repackage the .app into the signed
-# .dmg -- it needs no app-building tools and no Mac. The .app is staged
-# UNSIGNED (MACOS_CODESIGN_IDENTITY is unset) under a separate key; the tree
-# tarball above is unchanged (test jobs still consume it).
-if [[ "${OS}" == "macos" || "${OS}" == "macosnogpl" ]]; then
-    echo "--- [mac] Assemble the unsigned Julia.app"
-    # Pass the same MFLAGS as the main build (esp. TAGGED_RELEASE_BANNER): the
-    # contrib/mac/app rule re-runs binary-dist, and without the matching flags
-    # build_h.jl regenerates with a different banner, going stale and forcing a
-    # full system-image rebuild. With them it collapses to a fast re-install+re-tar.
-    MACOS_CODESIGN_IDENTITY="" ${MAKE} "${MFLAGS[@]}" -C contrib/mac/app "dmg/Julia-${MAJMIN?}.app"
-    tar zcf "${UPLOAD_FILENAME}.app.tar.gz" -C contrib/mac/app/dmg "Julia-${MAJMIN?}.app"
-    echo "--- [mac] Stage the unsigned .app to s3://${STAGING_TARGET}.app.tar.gz"
-    UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.app.tar.gz" "${STAGING_TARGET}.app.tar.gz"
-fi
+# macOS: the Julia.app is NOT assembled here. Since the app is now a pure
+# repackage of the .tar (contrib/mac/app needs no osacompile/plutil and no Mac),
+# it is assembled off the critical build->test path from the staged .tar instead:
+# julia-pr does it in a separate Linux app-builder (see stage_macos_app.sh), and
+# julia-ci does it in the trusted publish step (see upload_julia.sh). Both share
+# utilities/macos/assemble_app.sh. Keeping it out of this job shortens the build
+# that the test jobs wait on.

utilities/macos/assemble_app.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Assemble the UNSIGNED Julia.app from a julia binary-dist `.tar.gz`, WITHOUT
+# rebuilding julia and WITHOUT a Mac.
+#
+# The macOS build jobs no longer build the .app (see build_julia.sh); they only
+# stage the tree `.tar.gz`. The .app is now a pure repackage of that tarball --
+# contrib/mac/app's rule needs no osacompile/plutil and runs anywhere -- so both
+# consumers assemble it from the staged .tar on a Linux agent:
+#   * julia-pr  -> stage_macos_app.sh (a separate off-critical-path builder)
+#   * julia-ci  -> upload_julia.sh    (the trusted publish step, before signing)
+# Sharing one script keeps the bundle layout defined in exactly one place (the
+# julia repo's contrib/mac/app/Makefile).
+#
+# Usage:  assemble_app.sh <julia-tree.tar.gz>
+# Leaves the bundle at  contrib/mac/app/dmg/Julia-${MAJMIN}.app  (unsigned).
+#
+# Requires (exported by build_envs.sh, which the caller sources):
+#   MAKE, MAJMIN, JULIA_BINARYDIST_FILENAME
+# and a julia checkout as the current directory (JULIAHOME).
+set -euo pipefail
+
+SRC_TARBALL="${1:?usage: assemble_app.sh <julia-tree.tar.gz>}"
+
+: "${MAKE:?}" "${MAJMIN:?}" "${JULIA_BINARYDIST_FILENAME:?}"
+
+# The .app rule extracts $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz. Put the
+# tarball there under exactly that name; the name is only how the rule locates it
+# (the content is the macOS tree, regardless of the host that computed the name).
+if [[ "${SRC_TARBALL}" != "${JULIA_BINARYDIST_FILENAME}.tar.gz" ]]; then
+    cp -f "${SRC_TARBALL}" "${JULIA_BINARYDIST_FILENAME}.tar.gz"
+fi
+
+echo "--- [mac] Assemble the unsigned Julia-${MAJMIN}.app from ${SRC_TARBALL}"
+# MAKE=true rewrites the rule's `$(MAKE) -C $(JULIAHOME) binary-dist` into a no-op
+# `true ...`: we already hold the tarball and there is no built tree to re-dist
+# (a real `make binary-dist` on this fresh checkout would trigger a full build).
+# Empty MACOS_CODESIGN_IDENTITY leaves the bundle unsigned -- signing happens
+# later in julia-publish, or never for PR .apps.
+MACOS_CODESIGN_IDENTITY="" ${MAKE} -C contrib/mac/app MAKE=true "dmg/Julia-${MAJMIN}.app"

utilities/render_launch_pipeline.py

@@ -326,6 +326,15 @@ def render_arches_group_text(arches_file, yaml_file, group, allow_fail,
     ("build_macos.soft_fail.arches", "build_macos.yml"),
 ]
 
+# julia-pr-only: a Linux job per macOS arch that downloads the staged tree
+# tarball and repackages it into the unsigned .app (see stage_macos_app.sh).
+# Each rendered step carries `if: pipeline.slug == "julia-pr"`, so in julia-ci
+# (where publish assembles + signs the .app) these steps are emitted but skipped.
+# It depends only on the matching build_ job, so it sits OFF the build->test path.
+STAGE_APP_ARCHES = [
+    ("stage_macos_app.arches", "stage_macos_app.yml"),
+]
+
 ALLOW_FAIL_TEST_ARCHES = [
     ("test_linux.soft_fail.arches",   "test_linux.yml"),
     # PowerPC (test_linux.powerpc.soft_fail) intentionally omitted -- no-op on
@@ -451,6 +460,13 @@ def allow_fail_group_text():
     return emit_group("Allow Fail", "\n".join(c for c in chunks if c))
 
 
+def stage_app_group_text():
+    chunks = []
+    for arches, yml in STAGE_APP_ARCHES:
+        chunks.append(render_arches_group_text(arches, yml, "Stage App", "false"))
+    return emit_group("Stage App", "\n".join(c for c in chunks if c))
+
+
 # Trailing barrier + trigger of the trusted julia-publish pipeline (inlined
 # verbatim so the wait reliably barriers all dynamically-uploaded jobs).
 TRAILER = '''\
@@ -470,6 +486,7 @@ def main():
         check_group_text(),
         test_group_text(),
         allow_fail_group_text(),
+        stage_app_group_text(),
     ]
 
     # JuliaSyntax: gated on ./JuliaSyntax/Project.toml existing in the julia

utilities/stage_macos_app.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# julia-pr ONLY: assemble the unsigned macOS Julia.app from an already-built
+# tree tarball and stage it, on a small Linux agent that sits OFF the
+# build->test critical path.
+#
+# Why a separate builder: the macOS build jobs stage only the tree `.tar.gz`
+# (they no longer build the .app, see build_julia.sh), and julia-pr has no
+# trusted publish step to repackage it (that path is julia-ci-only). juliaup
+# resolves PR builds out of the ephemeral-pr bucket, so PR .apps still need to
+# land there -- but assembling them must not delay the test jobs, which depend
+# only on the build. This job depends on the macOS build and runs in parallel
+# with the tests.
+#
+# Requires TRIPLET (a *-apple-darwin triplet) to be defined.
+set -euo pipefail
+
+# Get UPLOAD_FILENAME, STAGING_TARGET, MAJMIN, MAKE, JULIA_BINARYDIST_FILENAME, ...
+# shellcheck source=SCRIPTDIR/build_envs.sh
+source .buildkite/utilities/build_envs.sh
+
+echo "--- Download the staged tree tarball artifact from build_${TRIPLET}"
+# The macOS build uploaded ${UPLOAD_FILENAME}.tar.gz as a buildkite artifact;
+# depends_on build_${TRIPLET} guarantees it exists.
+buildkite-agent artifact download "${UPLOAD_FILENAME}.tar.gz" . --step "build_${TRIPLET?}"
+
+# Assemble contrib/mac/app/dmg/Julia-${MAJMIN}.app (unsigned) from the tarball.
+# shellcheck source=SCRIPTDIR/macos/assemble_app.sh
+.buildkite/utilities/macos/assemble_app.sh "${UPLOAD_FILENAME}.tar.gz"
+
+echo "--- Pack the .app"
+tar zcf "${UPLOAD_FILENAME}.app.tar.gz" -C contrib/mac/app/dmg "Julia-${MAJMIN?}.app"
+
+echo "--- Stage the unsigned .app to s3://${STAGING_TARGET}.app.tar.gz"
+# Same write-once staging as the build job: assume the untrusted `stage` role
+# and upload under this build's commit-sha-gated path in the ephemeral-pr bucket.
+# shellcheck source=SCRIPTDIR/aws_oidc.sh
+source .buildkite/utilities/aws_oidc.sh stage
+# shellcheck source=SCRIPTDIR/upload_to_s3.sh
+source .buildkite/utilities/upload_to_s3.sh
+UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.app.tar.gz" "${STAGING_TARGET}.app.tar.gz"

utilities/upload_julia.sh

@@ -77,15 +77,18 @@ aws s3 cp "s3://${STAGING_TARGET}.tar.gz" "${UPLOAD_FILENAME}.tar.gz"
 # These are the extensions that we will always upload
 UPLOAD_EXTENSIONS=( "tar.gz" )
 
-# macOS: the build_ step staged an assembled (unsigned) Julia.app. Unpack it,
-# codesign everything in it (the launcher + the bundled julia tree), then derive
-# both products from the signed bundle: the .tar.gz (the tree) and the .dmg.
+# macOS: the build_ step no longer stages an assembled .app -- it stages only
+# the tree .tar.gz (downloaded above). Assemble the unsigned Julia.app from that
+# tarball here (the same repackage the julia-pr app-builder does; shared via
+# assemble_app.sh), codesign everything in it (the launcher + the bundled julia
+# tree), then derive both products from the signed bundle: the .tar.gz (the
+# tree) and the .dmg.
 if [[ "${OS}" == "macos" || "${OS}" == "macosnogpl" ]]; then
     APP_NAME="Julia-${MAJMIN?}.app"
-    echo "--- [mac] Download + unpack the staged unsigned .app"
-    aws s3 cp "s3://${STAGING_TARGET}.app.tar.gz" "${UPLOAD_FILENAME}.app.tar.gz"
+    echo "--- [mac] Assemble the unsigned .app from the downloaded tarball"
+    .buildkite/utilities/macos/assemble_app.sh "${UPLOAD_FILENAME}.tar.gz"
     rm -rf "${APP_NAME}"
-    tar zxf "${UPLOAD_FILENAME}.app.tar.gz"
+    mv "contrib/mac/app/dmg/${APP_NAME}" "${APP_NAME}"
     chmod -R u+w "${APP_NAME}"
 
     # Signing is split across three steps because the stdlib pkgimage checksum