diff --git a/.buildkite/README.md b/.buildkite/README.md index e22aebcc7..3f5f89c08 100644 --- a/.buildkite/README.md +++ b/.buildkite/README.md @@ -1,3 +1,31 @@ # This directory not used by base Julia! -This directory is for performing CI on the `julia-buildkite` repository itself! It does not get used at all during the actual build for Julia itself. +This directory is for performing CI on the `julia-buildkite` repository +itself! It does not get used at all during the actual build for Julia +itself. + +## How the self-test works + +The `julia-buildkite-ci` Buildkite pipeline (untrusted; same cluster as +`julia-pr` / `julia-ci`) builds this repository's pull requests and +`main`. Its webUI configuration is the same "Launch pipelines" step as +the julia build pipelines (see `pipelines/main/0_webui.yml`), but the +pipeline's *repository* is `JuliaCI/julia-buildkite` -- so every job of +the build checks out this repository and runs `hooks/post-checkout`, +which: + +1. replaces the working directory with a `JuliaLang/julia` checkout + (`UPSTREAM_BRANCH`, default `master`; pinned once per build via + meta-data so all jobs test the same julia commit), and +2. pins the external-buildkite plugin's build meta-data to + `${BUILDKITE_COMMIT}` (this build's julia-buildkite commit), so every + launched job runs the proposed pipeline code. + +Trust model (see `ops/README.md`): the self-test pipeline holds no +secrets, tokens, or signing rights. Build jobs stage their artifacts +write-once to the pipeline's own ephemeral bucket +(`julialang-ephemeral-buildkite`) via its own OIDC role +(`julia-oidc-stage-buildkite`) -- separate from the buckets juliaup and +julia-publish consume, so a self-test build can never place anything a +production consumer would read. The julia-publish trigger is gated on +`pipeline.slug == "julia-ci"` and so never fires here. diff --git a/.buildkite/cryptic_repo_root/.buildkite b/.buildkite/cryptic_repo_root/.buildkite deleted file mode 120000 index c25bddb6d..000000000 --- a/.buildkite/cryptic_repo_root/.buildkite +++ /dev/null @@ -1 +0,0 @@ -../.. \ No newline at end of file diff --git a/.buildkite/cryptic_repo_root/README.md b/.buildkite/cryptic_repo_root/README.md deleted file mode 100644 index c0c323dd7..000000000 --- a/.buildkite/cryptic_repo_root/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This directory is used to fake out `cryptic`'s search algorithms. -Execute things like `verify_treehashes --repo-root=$(pwd)` from within this directory. diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index be42e58ad..985974ca1 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,18 +1,26 @@ #!/usr/bin/env bash +# Repository hook for the julia-buildkite SELF-TEST pipeline: its jobs +# check out this repository, and this hook then (1) swaps the working +# directory for a JuliaLang/julia checkout (pinned once per build via +# meta-data, so every job tests the same julia commit) and (2) points the +# external-buildkite plugin's meta-data at this build's commit -- so the +# whole build/test flow runs against the proposed julia-buildkite code. +# See .buildkite/README.md. + set -euo pipefail # Buildkite users can trigger new builds with these values overridden to # test specific julia gitshas UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/JuliaLang/julia.git}" -UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-release-1.13}" +UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-master}" -# All of our workers are required to provide a default for the julia cache dir -# We're going to cache repositories in here, just like buildkite itself would: -UPSTREAM_CACHE="${BUILDKITE_PLUGIN_JULIA_CACHE_DIR?}/repos/$(tr ':/.' '---' <<<"${UPSTREAM_URL}")" +# Cache repositories in the worker's julia cache dir (if it provides one), +# just like buildkite itself would: +UPSTREAM_CACHE="${BUILDKITE_PLUGIN_JULIA_CACHE_DIR:-}/repos/$(tr ':/.' '---' <<<"${UPSTREAM_URL}")" DISABLE_UPSTREAM_CACHE="false" -if [[ "$(uname)" == "FreeBSD" ]]; then +if [[ "$(uname)" == "FreeBSD" || -z "${BUILDKITE_PLUGIN_JULIA_CACHE_DIR:-}" ]]; then DISABLE_UPSTREAM_CACHE="true" fi @@ -96,6 +104,7 @@ if ! buildkite-agent meta-data exists BUILDKITE_JULIA_VERSION; then buildkite-agent meta-data set BUILDKITE_JULIA_BRANCH "${UPSTREAM_BRANCH}" fi -# Export S3_BUCKET and S3_BUCKET_PREFIX, to force our CI to upload to a different S3 target than the actual CI pipeline would. -export S3_BUCKET="julialang-ephemeral" -export S3_BUCKET_PREFIX="julia-buildkite-uploads/bin" +# No S3 redirection is needed here: staging-bucket selection is keyed on +# the pipeline slug (utilities/build_envs.sh), so self-test jobs stage to +# the pipeline's own ephemeral bucket, and IAM only grants this pipeline's +# OIDC identity that bucket anyway (see ops/terraform/iam.tf). diff --git a/.gitignore b/.gitignore index 7a9708ad2..e58d87649 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ cryptic-buildkite-plugin/ # Ignore IDE files .vscode/ +__pycache__/ + +# Terraform working directories (provider binaries); lock files are committed +.terraform/ +*.tfstate +*.tfstate.* diff --git a/Makefile b/Makefile index 79476060e..3e5e9eb93 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,2 @@ -export PATH := $(shell pwd)/cryptic-buildkite-plugin/bin:$(PATH) - -decrypt: - cd .buildkite/cryptic_repo_root && decrypt --repo-root=$$(pwd) --verbose - -sign_treehashes: - cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose - git commit -a -m "sign treehashes" - -verify_treehashes: - cd .buildkite/cryptic_repo_root && verify_treehashes --repo-root=$$(pwd) --verbose - print-%: echo "$*=$($*)" diff --git a/cryptic_repo_keys/.gitignore b/cryptic_repo_keys/.gitignore deleted file mode 100644 index 8d18931db..000000000 --- a/cryptic_repo_keys/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore the unencrypted repo_key -repo_key - -# Ignore any agent keys (public or private) we have stored -agent_key* diff --git a/cryptic_repo_keys/README.md b/cryptic_repo_keys/README.md deleted file mode 100644 index a66b6f926..000000000 --- a/cryptic_repo_keys/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## Cryptic repository keys - -This folder contains RSA-encrypted symmetric AES keys. -These are used by buildkite agents to decrypt the secrets embedded within this repository. -Each buildkite agent contains an RSA secret key that is used to unlock the symmetric AES key that was used to encrypt the secrets within this repository. -For more information, see the [`cryptic` buildkite plugin repository](https://github.com/JuliaCI/cryptic-buildkite-plugin). diff --git a/cryptic_repo_keys/repo_key.2297e5e7 b/cryptic_repo_keys/repo_key.2297e5e7 deleted file mode 100644 index 2ab9198b4..000000000 Binary files a/cryptic_repo_keys/repo_key.2297e5e7 and /dev/null differ diff --git a/devdocs/macos-developer-id.md b/devdocs/macos-developer-id.md index 281b992b2..93bfaa162 100644 --- a/devdocs/macos-developer-id.md +++ b/devdocs/macos-developer-id.md @@ -1,54 +1,40 @@ -## Expired certificates - -The main symptom will be that the upload job will be failing. The unlock -keychain step prints the status of the certificate and it will print something -like (`CSSMERR_TP_CERT_EXPIRED`) if the certificate is expired. - -### Update expired Apple Developer ID - -To replace the certificate you will need first a MacOS machine and to emit a new -certificate. - -1. To replace an expired Apple certificate, clone `julia-buildkite` repo and - clone https://github.com/JuliaCI/cryptic-buildkite-plugin in its root. - You will also need to have the private `agent.key`, and point towards it - by defining the `AGENT_PRIVATE_KEY_PATH` environment variable. -2. You can decrypt by running `make decrypt`. -3. Get the `macos_codesigning.keychain` file in the `secrets` directory and add - it to your local keychains with the Keychain Access app (File > Import - Items). -4. From that app delete the old certificate and add the new one (it's a `.cer` - file). -5. Test the certificate by right clicking and running both the general - evaluation and the codesigning one. -6. Update the identity in `MACOS_CODESIGN_IDENTITY` (You can find the identity - by doing `security find-identity -p codesigning $(PATH_TO_KEYCHAIN)/macos_codesigning.keychain` ). -7. You can also test it by running the codesign.sh script in this repo with - `./utilities/macos/codesign.sh --keychain ./secrets/macos_codesigning.keychain --identity $(NEW_IDENTITY) ./test` - with some executable. -8. Afterward reencrypt the keychain by running - `./cryptic-buildkite-plugin/bin/encrypt_file --private-key=$(INSERT_PRIVATE_KEY) --repo-key=$(INSERT_REPO_KEY) ./secrets/macos_codesigning.keychain` -9. Finally sign the repo with `make sign_treehashes` - -The `security` cli app is also useful for debugging and managing keychains. You -can find more information about it by running `man security` in the terminal. +# macOS Developer ID certificate +The Developer ID Application private key lives in AWS KMS +(`alias/julia-macos-codesigning`); macOS codesigning happens via +`rcodesign` with the AWS KMS backend (see `utilities/macos/rcodesign/`). +The certificate itself is public and committed at +`utilities/macos/developer_id.pem`. + +## Expired certificate + +The main symptom will be failing upload jobs (signature validation errors +or Apple rejecting the notarization). To renew: + +1. Generate a fresh CSR from the KMS key (no new key needed): + `ops/22_generate_macos_csr.sh` +2. Submit it at https://developer.apple.com/account/resources/certificates + (type: "Developer ID Application"), download the `.cer`. +3. Convert and commit: + `openssl x509 -inform DER -in developerID_application.cer -out utilities/macos/developer_id.pem` +4. Test locally (requires kms:Sign on the key): + `rcodesign sign --aws-kms-key alias/julia-macos-codesigning --aws-kms-certificate-file utilities/macos/developer_id.pem ` ## New agreements -It is also possible that just a new agreement is needed. In that case, you will -see the following error message returned by the `codesign` command: +It is also possible that just a new agreement is needed. In that case, you +will see the following error message during notarization: -> HTTP status code: 403. A required agreement is missing or has expired. This -> request requires an in-effect agreement that has not been signed or has -> expired. Ensure your team has signed the necessary legal agreements and that -> they are not expired. +> HTTP status code: 403. A required agreement is missing or has expired. +> This request requires an in-effect agreement that has not been signed or +> has expired. Ensure your team has signed the necessary legal agreements +> and that they are not expired. -In this case, it is sufficient to visit `developer.apple.com` and logging in +In this case, it is sufficient to visit `developer.apple.com` and log in using the Apple ID that is associated with the Apple Developer account. You will be prompted to accept the new agreement. After that, the build should succeed. -Note that currently, the Apple ID that's used to sign Julia binaries is owned -by JuliaHub, so you will need to get in touch with somebody from the +Note that currently, the Apple ID that's used to sign Julia binaries is +owned by JuliaHub, so you will need to get in touch with somebody from the organization to accept the agreement. diff --git a/devdocs/sign.md b/devdocs/sign.md deleted file mode 100644 index 19358c006..000000000 --- a/devdocs/sign.md +++ /dev/null @@ -1,36 +0,0 @@ -## Dependencies - -1. [shyaml](https://github.com/0k/shyaml) - -## Instructions - -If you are a maintainer, and you want to re-sign all of the signatures, here are the steps: - -### If you only have access to the _repository_ private key - -```shell -git clone git@github.com:JuliaCI/julia-buildkite.git -cd julia-buildkite -git checkout YOURINITIALS/YOUR-BRANCH-NAME -git clone https://github.com/JuliaCI/cryptic-buildkite-plugin.git -mv /path/to/my/repository/private/key ./cryptic_repo_keys/repo_key -make sign_treehashes -git push origin YOURINITIALS/YOUR-BRANCH-NAME -``` - -### If you have access to the _agent_ private key - -``` -git clone git@github.com:JuliaCI/julia-buildkite.git -cd julia-buildkite -git checkout YOURINITIALS/YOUR-BRANCH-NAME -git clone https://github.com/JuliaCI/cryptic-buildkite-plugin.git -cd cryptic-buildkite-plugin -git checkout main -cd .. -export AGENT_PRIVATE_KEY_PATH=/path/to/my/agent.key -make sign_treehashes -unset AGENT_PRIVATE_KEY_PATH -rm -f /path/to/my/agent.key -git push origin YOURINITIALS/YOUR-BRANCH-NAME -``` diff --git a/ops/20_export_gpg_pubkey.py b/ops/20_export_gpg_pubkey.py new file mode 100755 index 000000000..393132d4c --- /dev/null +++ b/ops/20_export_gpg_pubkey.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# This file is a part of Julia. License is MIT: https://julialang.org/license +"""Export the KMS-held release tarball signing key as an OpenPGP public key. + +The signing key is generated inside KMS (alias/julia-tarball-signing, +created by ops/terraform) and never leaves it. This script fetches the +RSA public half via `aws kms get-public-key`, wraps it in an OpenPGP v4 +public key packet + user ID, and obtains the positive self-certification +with a `kms:Sign` call -- producing an armored public key block that GPG +and friends accept. Commit the output (signing-pubkeys/tarball_signing.pub.asc) +and publish it as the Julia releases signing key; release signatures +made by utilities/kms_gpg_sign.py verify against it. + +The OpenPGP fingerprint covers the key creation timestamp. It defaults to +the KMS key's own CreationDate (kms:DescribeKey) -- an immutable, already- +pinned value that also matches what kms_gpg_sign.py's --public-key-from-kms +signing path derives -- so re-running reproduces the identical key block. +--created overrides it only to reproduce a key minted under a different +timestamp; a changed timestamp would mint a "new" identity for the same key. + +Usage: + 20_export_gpg_pubkey.py [--created YYYY-MM-DD] [--kms-key-id ALIAS] [-o FILE] + +For testing without KMS access, `--local-key key.pem` uses a local RSA +private key via openssl, exercising the identical construction path. + +Requires AWS credentials with kms:GetPublicKey + kms:Sign on the key. +""" + +import argparse +import os +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(SCRIPT_DIR, "..", "utilities")) + +import kms_gpg_sign as K # noqa: E402 + +# The DER/SPKI parsing, kms:GetPublicKey fetch, and --created parsing live in +# kms_gpg_sign.py (shared with its --public-key-from-kms signing mode). + +DEFAULT_KMS_KEY = "alias/julia-tarball-signing" +DEFAULT_UID = "Julia Release Signing Key " +DEFAULT_OUTPUT = os.path.join(SCRIPT_DIR, "..", "signing-pubkeys", "tarball_signing.pub.asc") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--created", + help="override the key creation time (unix timestamp or " + "YYYY-MM-DD UTC midnight); part of the key fingerprint. " + "Defaults to the KMS key's own CreationDate") + parser.add_argument("--uid", default=DEFAULT_UID, + help=f"user ID for the key (default: {DEFAULT_UID!r})") + parser.add_argument("-o", "--output", default=DEFAULT_OUTPUT, + help="output path for the armored public key block") + group = parser.add_mutually_exclusive_group() + group.add_argument("--kms-key-id", default=DEFAULT_KMS_KEY, + help=f"AWS KMS key ID/ARN/alias (default: {DEFAULT_KMS_KEY})") + group.add_argument("--local-key", + help="local RSA private key PEM (testing only)") + args = parser.parse_args() + + if args.created: + timestamp = K.parse_created(args.created) + elif args.local_key: + # A local test key has no KMS CreationDate; pin a fixed epoch so the + # construction path stays deterministic. + timestamp = 0 + else: + # Pin the fingerprint to the KMS key's immutable provisioning date, + # matching kms_gpg_sign.py's --public-key-from-kms signing path. + timestamp = K.kms_key_creation_date(args.kms_key_id) + + if args.local_key: + spki = K.local_public_key(args.local_key) + signer = K.LocalOpensslSigner(args.local_key) + else: + spki = K.kms_public_key(args.kms_key_id) + signer = K.KmsSigner(args.kms_key_id) + + n, e = K.rsa_components_from_spki(spki) + key_body = K.build_rsa_key_body(n, e, timestamp) + pubkey, cert = K.build_certificate(signer, key_body, args.uid, timestamp) + armored = K.armor(cert, "PUBLIC KEY BLOCK") + + with open(args.output, "w") as f: + f.write(armored) + + # Round-trip self-check: what we wrote parses back to the same key. + assert K.load_public_key(args.output).fingerprint == pubkey.fingerprint + + print(f"Fingerprint: {pubkey.fingerprint.hex().upper()}") + print(f"User ID: {args.uid}") + print(f"Created: {timestamp}") + print(f"Written to: {args.output}") + print() + print("Commit this file, and publish it as the Julia releases signing key") + print("(it replaces the pre-migration juliareleases.asc).") + + +if __name__ == "__main__": + main() diff --git a/ops/21_import_notary_key.sh b/ops/21_import_notary_key.sh new file mode 100755 index 000000000..edce99b72 --- /dev/null +++ b/ops/21_import_notary_key.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Import the App Store Connect API private key (.p8, ECDSA P-256) into the +# alias/julia-notary-api KMS key (EXTERNAL origin), then emit the unified +# api key JSON used by rcodesign (contains no secret material). +# +# Usage: 21_import_notary_key.sh AuthKey_XXXXXXXXXX.p8 +# +# Apple generates ASC API keys, so the existing key must be imported (we +# cannot register our own public key with Apple). Run from a trusted +# machine; securely delete the .p8 afterwards. +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +P8_FILE="${1:?usage: $0 AuthKey.p8 }" +ISSUER_ID="${2:?usage: $0 AuthKey.p8 }" +API_KEY_ID="${3:?usage: $0 AuthKey.p8 }" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# GetParametersForImport / ImportKeyMaterial reject aliases; resolve to the ARN. +KEY_ARN="$(kms_key_arn "${KMS_ALIAS_NOTARY_API}")" +if [ -z "${KEY_ARN}" ]; then + echo "ERROR: ${KMS_ALIAS_NOTARY_API} not found (run terraform apply first)" >&2 + exit 1 +fi + +echo "--- Convert .p8 (PEM PKCS#8) to DER" +openssl pkcs8 -topk8 -nocrypt -in "${P8_FILE}" -outform DER -out "${WORK}/key.pkcs8.der" + +echo "--- Get KMS import parameters" +aws kms get-parameters-for-import --region "${AWS_REGION}" \ + --key-id "${KEY_ARN}" \ + --wrapping-algorithm RSA_AES_KEY_WRAP_SHA_256 \ + --wrapping-key-spec RSA_4096 \ + --output json > "${WORK}/import-params.json" + +python3 - "${WORK}" <<'EOF' +import base64, json, sys +work = sys.argv[1] +params = json.load(open(f"{work}/import-params.json")) +open(f"{work}/wrapping_key.der", "wb").write(base64.b64decode(params["PublicKey"])) +open(f"{work}/import_token.bin", "wb").write(base64.b64decode(params["ImportToken"])) +EOF + +echo "--- Wrap key material" +openssl rand 32 > "${WORK}/aes.key" +openssl pkeyutl -encrypt \ + -pubin -inkey "${WORK}/wrapping_key.der" -keyform DER \ + -pkeyopt rsa_padding_mode:oaep \ + -pkeyopt rsa_oaep_md:sha256 \ + -pkeyopt rsa_mgf1_md:sha256 \ + -in "${WORK}/aes.key" -out "${WORK}/wrapped_aes.bin" +openssl enc -id-aes256-wrap-pad \ + -K "$(xxd -p "${WORK}/aes.key" | tr -d '\n')" \ + -iv A65959A6 \ + -in "${WORK}/key.pkcs8.der" -out "${WORK}/wrapped_key.bin" +cat "${WORK}/wrapped_aes.bin" "${WORK}/wrapped_key.bin" > "${WORK}/encrypted_material.bin" + +echo "--- Import key material into ${KMS_ALIAS_NOTARY_API}" +aws kms import-key-material --region "${AWS_REGION}" \ + --key-id "${KEY_ARN}" \ + --encrypted-key-material "fileb://${WORK}/encrypted_material.bin" \ + --import-token "fileb://${WORK}/import_token.bin" \ + --expiration-model KEY_MATERIAL_DOES_NOT_EXPIRE + +OUT_JSON="${SCRIPT_DIR}/../utilities/macos/notary_api_key.json" + +echo "--- Write unified api key JSON (no secret material) to ${OUT_JSON}" +cat > "${OUT_JSON}" < /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +RCODESIGN="${RCODESIGN:-rcodesign}" + +"${RCODESIGN}" generate-certificate-signing-request \ + --aws-kms-key "${KMS_ALIAS_MACOS_CODESIGN}" \ + --aws-kms-region "${AWS_REGION}" \ + --csr-pem-file julia-macos-codesigning.csr.pem + +echo "CSR written to julia-macos-codesigning.csr.pem" diff --git a/ops/23_put_tokens.sh b/ops/23_put_tokens.sh new file mode 100755 index 000000000..7908cb562 --- /dev/null +++ b/ops/23_put_tokens.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Store a CI telemetry bearer token in SSM Parameter Store. +# +# Bearer tokens (codecov, coveralls, buildkite analytics) are inherently +# symmetric secrets and cannot be turned into KMS signing operations, so +# they live in the AWS secrets store and are fetched at job runtime by +# the julia-oidc-tokens-ci role (julia-ci only; PR builds get no +# tokens). Nothing secret is stored in the repository. +# +# Usage: +# 23_put_tokens.sh codecov_token # prompts for the value +# 23_put_tokens.sh coveralls_token +# 23_put_tokens.sh buildkite_analytics_token +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +NAME="${1:?usage: $0 }" + +if [[ ! "${NAME}" =~ ^[a-z0-9_]+$ ]]; then + echo "ERROR: token name must be lowercase [a-z0-9_]" >&2 + exit 1 +fi + +read -r -s -p "Value for ${SSM_TOKEN_PREFIX}/${NAME}: " VALUE +echo + +aws ssm put-parameter --region "${AWS_REGION}" \ + --name "${SSM_TOKEN_PREFIX}/${NAME}" \ + --type SecureString \ + --value "${VALUE}" \ + --overwrite + +echo "Stored ${SSM_TOKEN_PREFIX}/${NAME}" diff --git a/ops/24_docs_deploy_pubkey.sh b/ops/24_docs_deploy_pubkey.sh new file mode 100755 index 000000000..ad718992f --- /dev/null +++ b/ops/24_docs_deploy_pubkey.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Print the SSH public key of the KMS-held docs deploy key, in OpenSSH +# format. Register this as a deploy key (with write access) on +# github.com/JuliaLang/docs.julialang.org. +# +# The private half never leaves KMS: the docs deploy job's ssh uses the +# aws-kms-pkcs11 provider (https://github.com/JackOfMostTrades/aws-kms-pkcs11) +# so each SSH authentication is a kms:Sign call gated by the +# julia-oidc-docs-deploy role. +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +aws kms get-public-key --region "${AWS_REGION}" \ + --key-id "${KMS_ALIAS_DOCS_DEPLOY}" \ + --query PublicKey --output text | base64 -d > "${WORK}/spki.der" + +openssl pkey -pubin -inform DER -in "${WORK}/spki.der" -out "${WORK}/pub.pem" +ssh-keygen -i -m PKCS8 -f "${WORK}/pub.pem" > "${WORK}/id.pub" + +echo "$(cat "${WORK}/id.pub") julia-docs-deploy@kms" diff --git a/ops/32_gen_test_codesign_cert.sh b/ops/32_gen_test_codesign_cert.sh new file mode 100755 index 000000000..e9e483308 --- /dev/null +++ b/ops/32_gen_test_codesign_cert.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Generate a SELF-ISSUED (throwaway) macOS codesigning certificate for the +# NON-PRODUCTION test KMS key alias/julia-macos-codesigning-test, and commit it +# as utilities/macos/developer_id_test.pem. +# +# This is the test-stack counterpart of ops/22_generate_macos_csr.sh. Unlike +# production -- where the CSR is submitted to Apple and the issued Developer ID +# certificate is committed as developer_id.pem -- here we never involve Apple: +# we mint a CSR from the test KMS key (which carries the key's PUBLIC half, +# self-signed by KMS to prove possession), then issue a leaf certificate for it +# from a throwaway local CA. The resulting cert therefore contains the test KMS +# key's public key, so rcodesign signatures made with that KMS key verify +# against it. It does NOT chain to Apple and Gatekeeper would reject it -- but +# the test stack only exercises the SIGNING mechanics (rcodesign -> kms:Sign), +# not Gatekeeper acceptance, and macOS notarization is skipped entirely +# (PUBLISH_SKIP_NOTARIZATION=1). +# +# One-time: run with AWS credentials that have kms:GetPublicKey + kms:Sign on +# the test key (e.g. after `terraform apply` with buildkite_test_pipeline_id +# set), then commit the output. Requires the patched rcodesign and openssl. +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +RCODESIGN="${RCODESIGN:-rcodesign}" +OUT="${SCRIPT_DIR}/../utilities/macos/developer_id_test.pem" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# 1. CSR from the test KMS key (carries its public key; signed by KMS). +"${RCODESIGN}" generate-certificate-signing-request \ + --aws-kms-key "${KMS_ALIAS_MACOS_CODESIGN_TEST}" \ + --aws-kms-region "${AWS_REGION}" \ + --csr-pem-file "${WORK}/kms.csr" + +# 2. Throwaway local CA (its private key is discarded with $WORK). +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "${WORK}/ca.key" -out "${WORK}/ca.pem" -days 3650 -sha256 \ + -subj "/CN=Julia TEST codesigning CA (NOT FOR PRODUCTION)/O=The Julia Project (TEST)/C=US" + +# 3. Issue the leaf cert for the KMS public key, with the codesigning EKU. +cat > "${WORK}/leaf.ext" <<'EOF' +keyUsage = critical, digitalSignature +extendedKeyUsage = critical, codeSigning +basicConstraints = critical, CA:FALSE +EOF +openssl x509 -req -in "${WORK}/kms.csr" \ + -CA "${WORK}/ca.pem" -CAkey "${WORK}/ca.key" -CAcreateserial \ + -days 3650 -sha256 -extfile "${WORK}/leaf.ext" \ + -out "${OUT}" + +echo "Wrote self-issued test codesigning certificate to ${OUT}" +openssl x509 -in "${OUT}" -noout -subject -issuer -dates +echo +echo "Commit ${OUT} (a public, non-secret, non-production certificate)." diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 000000000..65963fa90 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,329 @@ +# Julia CI trust infrastructure (post-cryptic) + +This directory configures the AWS (and Azure) resources that replace the +`cryptic` Buildkite plugin: + +- **`terraform/`** — the declarative infrastructure: Buildkite OIDC + provider, the four KMS keys, the IAM roles and their policies. +- **`terraform/azure/`** — a separate root module (different control + plane / credentials) for the Azure Trusted Signing federated identity + credentials. +- **numbered scripts** — the imperative one-time operations: importing + existing key material into KMS, entering bearer tokens, generating the + macOS CSR, printing the docs deploy public key, uploading tool binaries. + These stay out of Terraform on purpose: their inputs are secrets, and + anything Terraform touches ends up in its state file. The Terraform + state for this module therefore contains **no secret material** and can + be stored in any ordinary backend. After this migration, **no private key ever +exists on a build agent or in this repository** — every signature is a +remote KMS operation, authorized by the job's OIDC identity: + +| Concern | Before (cryptic) | After | +|--------------------------|-------------------------------------------|--------------------------------------------------------------| +| S3 uploads | static `AWS_ACCESS_KEY_ID/SECRET` in yml | OIDC → `julia-oidc-stage-{pr,ci}` (untrusted, per pipeline) + `julia-oidc-publish` (trusted) roles | +| macOS codesigning | keychain file w/ Developer ID key | KMS RSA key + patched `rcodesign` (`utilities/macos/rcodesign`) | +| macOS notarization | Apple ID + app-specific password | App Store Connect API key in KMS (ES256 JWTs via `kms:Sign`) | +| Linux/source GPG signing | raw GPG private key file | fresh key generated in KMS, `utilities/kms_gpg_sign.py` (new public key published) | +| Windows codesigning | `AZURE_CLIENT_SECRET` | Azure workload identity federation (Buildkite OIDC) | +| Docs deploy SSH key | cryptic-encrypted key file | SSH key in KMS, ssh signs via [aws-kms-pkcs11](https://github.com/JackOfMostTrades/aws-kms-pkcs11) | +| Telemetry bearer tokens | cryptic-encrypted variables | SSM Parameter Store (SecureString), OIDC-gated `ssm:GetParameter` | + +(The codecov / coveralls / buildkite-analytics tokens are bearer tokens — +there is no public-key operation to delegate — so they live in the AWS +secrets store and are fetched at runtime; they are never stored in the +repository in any form.) + +## Trusted / untrusted pipeline split + +The single most important control is that **pull requests and release +publishing run in different Buildkite pipelines**, and the trusted IAM roles +only trust the publish pipeline's slug. There are three pipelines: + +- **`julia-pr`** — builds pull requests (untrusted). +- **`julia-ci`** — builds trusted refs only: master, release-*, tags, and the + scheduled nightlies (untrusted to sign, but it is what triggers publish). +- **`julia-publish`** — the trusted pipeline that signs + promotes. + +A fourth, untrusted pipeline, **`julia-buildkite-ci`**, is the +julia-buildkite repository's own self-test CI (see `.buildkite/README.md`): +it builds julia with *proposed* pipeline code, staging write-once to its own +bucket (`julialang-ephemeral-buildkite`, role `julia-oidc-stage-buildkite`) +that neither juliaup nor julia-publish ever reads. It gets no tokens and no +publish trigger, exactly like `julia-pr`. + +``` + julia-pr (pull requests) julia-ci (master / release-* / tags / scheduled) + build ──► s3://julialang-ephemeral-pr/ build ──► s3://julialang-ephemeral-ci/ + //julia-* //julia-* + (UNTRUSTED: the build step stages directly -- write-once, own pipeline's + ephemeral bucket, own commit's path; per-pipeline roles + julia-oidc-stage-pr / julia-oidc-stage-ci, no KMS) + PRs stop here (juliaup reads │ tests pass, julia-ci only: + the -pr bucket). ▼ trigger + julia-publish ──► publish_all (single step) + (TRUSTED: role julia-oidc-publish, kms:Sign + read julia-ci staging bucket + ONLY + write final) + verify_trusted_commit.sh → sign (rcodesign / Trusted Signing / KMS-GPG) → promote → deploy docs +``` + +Why this is safe where branch-pinning was not: Buildkite reports a pull +request build's `sub` ref as the PR head branch, with no PR-vs-push +discriminator, so a fork PR whose branch is named `master` would match a +`ref:refs/heads/master` trust pattern. We therefore do **not** trust any +build-pipeline slug for signing. The trusted roles trust only the +`julia-publish` slug, and a PR (which only ever runs in `julia-pr`) cannot +produce a build under that slug. + +**Required Buildkite configuration** (this is load-bearing — the IAM trust +depends on it): +- `julia-pr`: builds pull requests (this is the only pipeline that should). +- `julia-ci`: builds master / release-* / tags / schedule; **does not build + pull requests**. +- `julia-publish`: "Build pull requests" OFF (incl. third-party forks); + branch-limited to `master release-*` (plus build tags `v*`); triggered only + by `julia-ci`. Its webUI step loads launch steps from a pinned + julia-buildkite (the external-buildkite plugin), never from the triggered + build's tree. +- Backstop: every publish job runs `utilities/verify_trusted_commit.sh`, + which aborts unless `BUILDKITE_COMMIT` is reachable from a protected ref + of the canonical upstream — so even a mis-triggered build cannot publish. + +The publish step runs on a single linux `queue: publish` agent: every +signature is remote-key (KMS / Trusted Signing) and all packaging tooling +is ported to linux (see "Publish image prerequisites" below). The macOS +Julia.app is assembled by the build_ step (`make -C contrib/mac/app`, which +already runs on a Mac so `osacompile` is available) and staged unsigned; the +publish step only codesigns it and wraps it in the `.dmg`, so no Mac is +needed there. + +## Trust model + +Buildkite agents mint OIDC tokens (`buildkite-agent oidc request-token`) +whose `sub` claim is `organization::pipeline::ref::commit::step:` +and which carry AWS session tags (`step_key`, `build_commit`, `pipeline_slug`, ...). + +* **Trusted roles** (`julia-oidc-publish`, `julia-oidc-docs-deploy`) are + assumable only from the `julia-publish` pipeline slug, and only from the + expected step (`aws:RequestTag/step_key` in the trust policy). +* Every trust policy additionally pins the Buildkite **organization, + pipeline, and cluster UUIDs** (`aws:RequestTag/organization_id` / + `pipeline_id` / `cluster_id`, values in + `ops/terraform/buildkite_ids.auto.tfvars`). Slugs are renameable and can + be re-minted by deleting + recreating a pipeline; the UUIDs cannot, so a + recreated pipeline with a matching slug does not regain role access. + IAM can only condition on `aud`/`sub` from the raw OIDC token, which is + why these (like `step_key`) travel as AWS session tags requested in + `utilities/aws_oidc.sh`; the claims are attested by Buildkite, and a + token requested without the tags fails the trust conditions outright. +* **Staging**: each build pipeline has its own role + (`julia-oidc-stage-pr` / `julia-oidc-stage-ci`), assumable from any job + of that one pipeline (the build step assumes it directly; there is no + step restriction on the untrusted roles). Each role can *only* write to + its own pipeline's staging bucket, and only below + `/${aws:PrincipalTag/build_commit}/*` — the source git sha, + an attested session tag the job cannot influence. The buckets + (`julialang-ephemeral-pr` / `julialang-ephemeral-ci`) are ephemeral and + lifecycle-expired, and publish reads **only** the `julia-ci` one — so a + PR build can never place, or pre-claim (paths are write-once), anything + that publish would consume. Build / PR jobs cannot touch release paths + or sign anything. (Consumers, e.g. juliaup, map PR number → head sha + via the GitHub API and fetch from the sha path in the `-pr` bucket.) +* **Tokens**: only `julia-ci` has a tokens role (`julia-oidc-tokens-ci`, + SSM `ssm:GetParameter` on the telemetry tokens). There is deliberately + no `-pr` counterpart: a pull request executes attacker-controlled code + inside the job, which could exfiltrate any bearer token the job can + read — so PR builds hold **no tokens at all** (and consequently no + coverage/analytics uploads happen on PRs). +* **No overwrites**: all roles must use S3 conditional writes + (`If-None-Match: *`, enforced via the `s3:if-none-match` policy condition); + uploads of already-existing objects fail. The only exception is the + `julia-latest-*` pointer objects, which release builds intentionally + repoint. Object versioning on the bucket is recommended belt-and-braces. +* Signing never exposes key material: every signature (macOS code signature, + notarization JWT, GPG tarball signature, docs-deploy SSH authentication) + is a `kms:Sign` call, conditioned on the calling job's step + (`aws:PrincipalTag/step_key`). + +## Runbook + +One-time setup, in order (admin AWS credentials; region and bucket names +are Terraform variables with the production defaults): + +1. Create the four Buildkite pipelines and set their WebUI steps: + - `julia-pr` — builds pull requests; WebUI = `pipelines/main/0_webui.yml`. + - `julia-ci` — builds master / release-* / tags / schedule (no PRs); + WebUI = `pipelines/main/0_webui.yml` (same launch flow; the rendered + publish trigger is gated on `pipeline.slug == "julia-ci"`, see + `utilities/render_launch_pipeline.py`). + - `julia-publish` — PRs OFF, branch-limited, triggered by `julia-ci`; + WebUI = `pipelines/publish/0_webui.yml`. + - `julia-buildkite-ci` — repository `JuliaCI/julia-buildkite`, PRs ON + (incl. forks), branch builds limited to `main`; WebUI = + `pipelines/main/0_webui.yml` again (the repository hook + `.buildkite/hooks/post-checkout` does the julia checkout swap). + All are plain `buildkite-agent pipeline upload` (no cryptic plugin, no + `cryptic_capable` agent targeting). +2. Record the organization / pipeline / cluster UUIDs that the IAM trust + policies pin (in addition to the slug-based `sub` patterns; slugs can + be renamed or re-minted, UUIDs cannot) in + `ops/terraform/buildkite_ids.auto.tfvars` and commit it (the UUIDs are + not secrets). They come from the Buildkite REST API (token scope + `read_pipelines`): `GET /v2/organizations/julialang` (`.id`) and + `GET /v2/organizations/julialang/pipelines/` (`.id`, + `.cluster_id`). The UUIDs are static; this needs redoing only if a + pipeline is ever recreated or moved to another cluster. Terraform + refuses to apply without real values here. +3. `terraform -chdir=ops/terraform init && terraform -chdir=ops/terraform apply` + — creates the OIDC provider for `agent.buildkite.com`, the four KMS + keys (the notary key as `EXTERNAL`-origin, pending import), and the + IAM roles + policies (one stage role per build pipeline, tokens for + julia-ci only, publish + docs-deploy). Re-apply any time trust + patterns or policies change. Local state is fine while testing (it + contains no secrets); once things settle, move it to S3 by adding a + `backend "s3"` block (bucket `julia-ci-tfstate`, `use_lockfile = + true`) and running `terraform init -migrate-state` — the resources + are untouched by the migration. +4. Key material: + * `./20_export_gpg_pubkey.py` — exports the OpenPGP public half of the + KMS-generated tarball signing key to + `signing-pubkeys/tarball_signing.pub.asc`. Commit it, and publish it as the + new Julia releases signing key (it **replaces** the pre-migration + `juliareleases.asc`; old signatures keep verifying against the old + key, new signatures only against this one). The creation timestamp is + part of the key fingerprint; it defaults to the KMS key's own + immutable CreationDate, so no date need be pinned by hand. + * `./21_import_notary_key.sh AuthKey_X.p8 ` — the + App Store Connect API key is Apple-generated and must be imported + (from a trusted workstation; obtain the .p8 once via the legacy + cryptic agent key, securely delete it afterwards). Writes + `utilities/macos/notary_api_key.json` — commit it; it contains no + secret material. +5. Telemetry tokens into SSM: + * `./23_put_tokens.sh codecov_token` + * `./23_put_tokens.sh coveralls_token` + * `./23_put_tokens.sh buildkite_analytics_token` +6. macOS certificate for the new KMS key: + * `./22_generate_macos_csr.sh` + * Submit CSR at developer.apple.com → Developer ID Application cert + * `openssl x509 -inform DER -in developerID_application.cer -out utilities/macos/developer_id.pem` + and commit (certificates are public). +7. Docs deploy key: + * `./24_docs_deploy_pubkey.sh` and register the printed key as a + deploy key with write access on JuliaLang/docs.julialang.org. + * Ensure the `aws_uploader` rootfs image (JuliaCI/rootfs-images) + ships `aws_kms_pkcs11.so` (https://github.com/JackOfMostTrades/aws-kms-pkcs11). +8. rcodesign (apple-codesign + AWS KMS backend) comes from `rcodesign_jll`, + built by Yggdrasil (`R/rcodesign`) from JuliaCI/apple-platform-rs branch + `julia-build`. To ship a new build: push `julia-build`, bump the + `GitSource` commit in the Yggdrasil recipe, then update the version / + sha256 / URL pins in `utilities/macos/get_rcodesign.sh` + (see `utilities/macos/rcodesign/README.md`). + (The macOS Julia.app is built + staged by the build_ step on the Mac + builder; the publish step only codesigns + packages it, so there is no + committed `.app` skeleton anymore.) +9. `terraform -chdir=ops/terraform/azure apply -var azure_app_id=` + (with Azure credentials that may manage the Trusted Signing app + registration) — federated credentials for Windows Trusted Signing + (matched to the `julia-publish` pipeline, where Windows signing now + runs); fill the (non-secret) `AZURE_TENANT_ID` / `AZURE_CLIENT_ID` + placeholders on the `publish_all` step in `pipelines/publish/launch.yml`. + If flexible federated credentials are unavailable on the tenant, fall + back to `--subject-claim organization_id` tokens (exact-match credential + on the Buildkite organization UUID) at the cost of org-level granularity. +10. Fill `JULIA_CI_AWS_ACCOUNT_ID` in `utilities/aws_oidc.sh` (from the + `julia_ci_aws_account_id` Terraform output). +11. Once green: revoke the legacy static AWS IAM user, delete the cryptic + agent keys from the agents, decommission `cryptic_capable` queues, + turn off webhook builds on the legacy `julia-buildkite` / + `julia-buildkite-scheduled` self-test pipelines (superseded by + `julia-buildkite-ci`) and retire the old `julialang-ephemeral` + self-test bucket, + revoke the old Apple Developer ID certificate, the Apple ID + app-specific password, the old SSH deploy key, and the + `AZURE_CLIENT_SECRET`. Revoke the old GPG release signing key (its + private half lived on agents) and securely delete all copies; keep + its revoked public key published so old releases remain verifiable. + +### NON-PRODUCTION publish test stack (`julia-publish-test-nosecrets`) + +An isolated stack to debug the publish flow (download → KMS-sign → promote) +end to end without touching any production key, bucket, or `julia-latest-*` +pointer. It runs the *same* `publish.sh` but redirects every signer to a +throwaway `*-test` KMS key (self-signed cert) and every bucket to +`julialang-test-publish`, and skips the two signers with no non-production +equivalent (Windows Azure Trusted Signing, macOS Apple notarization). The +macOS `.dmg` is still built and codesigned. All of it lives in +`ops/terraform/test_publish.tf` (gated on `buildkite_test_pipeline_id`) and +`pipelines/publish-test/`. + +1. Create the `julia-publish-test-nosecrets` Buildkite pipeline (WebUI = + `pipelines/publish-test/0_webui.yml`); it is NOT a trust boundary, so PR + builds may stay enabled. Put its pipeline + cluster UUIDs in + `buildkite_ids.auto.tfvars` (`buildkite_test_pipeline_id` / + `buildkite_test_cluster_id`). +2. `terraform apply` — creates the two test keys (`alias/julia-{tarball-signing, + macos-codesigning}-test`), the `julialang-test-publish` bucket, and the + `julia-oidc-publish-test` role (test keys + test bucket only, no write-once). +3. Generate + commit the macOS test cert (AWS creds with kms:Sign on the test + key, plus a built rcodesign): + * `./32_gen_test_codesign_cert.sh` → `utilities/macos/developer_id_test.pem` + The GPG test pubkey is NOT committed: the test pipeline sets + `TARBALL_SIGNING_PUBKEY=""` and derives the key identity from KMS at runtime + (`kms_gpg_sign.py --public-key-from-kms`, using `kms:GetPublicKey` for the + RSA public half and `kms:DescribeKey` for the key's creation date). +4. Stage real input: run a `julia-ci` (or `julia-pr`) build so it stages the + unsigned tarballs (and the unsigned macOS Julia.app) for some commit into + its ephemeral bucket. The test pipeline reads `STAGING_BUCKET` + (`julialang-ephemeral-ci`, public-read) and only PROMOTES to the test + bucket, so no seeding is needed. +5. Trigger a `julia-publish-test-nosecrets` build on that same commit and watch + it download the staged artifacts → KMS-sign (test keys) → build the `.dmg` → + promote into `julialang-test-publish`. Re-run freely (the test role has no + write-once condition). Tear down by deleting `ops/terraform/test_publish.tf`. + +## Agent/rootfs prerequisites + +* AWS CLI on **all build agents/rootfs images** (the build step itself + stages to S3 now), recent enough for + `aws s3api put-object --if-none-match` (conditional writes). For linux + builds that means inside the build rootfs images (`package_*`); also the + macOS / Windows / FreeBSD build agents. +* AWS CLI on test agents/rootfs images too (the test step fetches the + Test Analytics token from SSM itself). This one is soft: test_julia.sh + skips the analytics upload with a warning when `aws` is missing. +* `aws_kms_pkcs11.so` in the `aws_uploader` rootfs (docs deploy). + +### Publish image prerequisites (linux, `queue: publish`) + +The single publish step signs and packages for every OS on linux: + +* AWS CLI; python3 (stdlib only; `kms_gpg_sign.py`, plist editing, PE + signature checks); a host julia (provided by the JuliaCI/julia plugin; + patches pkgimage checksums in the foreign install trees). +* macOS: `rcodesign` is fetched at runtime (pinned sha256, see + `utilities/macos/get_rcodesign.sh`); for the `.dmg`: `mkfs.hfsplus` + (hfsprogs) and the `hfsplus` + `dmg` tools from + [mozilla/libdmg-hfsplus](https://github.com/mozilla/libdmg-hfsplus) + (the same tools Mozilla uses to package Firefox DMGs on linux; the + mozilla fork carries the `symlink` and `attr` subcommands we use). +* Windows: Wine (64-bit) with Inno Setup 6 installed in the prefix at + `C:\Program Files (x86)\Inno Setup 6` (or set `ISCC_EXE`); `jsign` + >= 6.0 on PATH (Azure Trusted Signing storetype) + a JRE; `7z` (p7zip) + for the .zip. + +## Notes + +* The GPG tarball signing key is **generated inside KMS** and never + exists anywhere else; a new public key must therefore be published + (`ops/20_export_gpg_pubkey.py`), and signatures made after the + migration do not verify against the pre-migration `juliareleases.asc`. + Only the notary key uses KMS `EXTERNAL` (BYOK) import, because Apple + generates App Store Connect API keys and we cannot register our own + public key with Apple. +* Retried upload jobs hitting an existing identical object are handled in + `utilities/upload_julia.sh` (412 + ETag comparison), not by allowing + overwrites. +* juliaup needs a change to consume PR binaries: resolve PR number → head + sha via the GitHub API, then fetch + `s3://julialang-ephemeral-pr/bin//julia-*` (this replaces the old + `julia-prNNNN-` upload filenames). diff --git a/ops/common.sh b/ops/common.sh new file mode 100755 index 000000000..9cb3e8bd5 --- /dev/null +++ b/ops/common.sh @@ -0,0 +1,45 @@ +# Shared configuration for the julia-buildkite AWS ops scripts. +# Source this from the numbered scripts; do not run directly. +# +# These scripts cover the imperative one-time operations (key material +# import, secret entry, CSR generation) that deliberately do NOT live in +# Terraform: their inputs are secrets that must never enter the Terraform +# state. The declarative infrastructure (OIDC provider, KMS keys, IAM +# roles, Azure federated credentials) is managed in ops/terraform/. +# +# All scripts are idempotent: safe to re-run after partial failures. + +set -euo pipefail + +# --- Configuration (override via environment) -------------------------------- +# Must match the corresponding Terraform variables (ops/terraform/variables.tf). + +export AWS_REGION="${AWS_REGION:-us-east-1}" + +# S3 bucket that release binaries (and prebuilt CI tools) are uploaded to +export S3_BUCKET="${S3_BUCKET:-julialangnightlies}" + +# KMS key aliases (created by ops/terraform) +export KMS_ALIAS_MACOS_CODESIGN="alias/julia-macos-codesigning" +export KMS_ALIAS_NOTARY_API="alias/julia-notary-api" +export KMS_ALIAS_TARBALL_SIGNING="alias/julia-tarball-signing" +export KMS_ALIAS_DOCS_DEPLOY="alias/julia-docs-deploy" + +# Throwaway NON-PRODUCTION test keys (created by ops/terraform/test_publish.tf +# only when the test pipeline UUID is set). Used by the isolated +# julia-publish-test-nosecrets debug stack with self-signed material. +export KMS_ALIAS_MACOS_CODESIGN_TEST="alias/julia-macos-codesigning-test" +export KMS_ALIAS_TARBALL_SIGNING_TEST="alias/julia-tarball-signing-test" + +# SSM Parameter Store prefix for CI telemetry bearer tokens (codecov, +# coveralls, buildkite analytics). These cannot be public-key operations, +# so they live in the AWS secrets store, fetched at runtime via OIDC. +export SSM_TOKEN_PREFIX="/julia-ci/tokens" + +# --- Helpers ------------------------------------------------------------------ + +# kms_key_arn -> key ARN (empty if alias doesn't exist) +kms_key_arn() { + aws kms describe-key --key-id "$1" --region "${AWS_REGION}" \ + --query KeyMetadata.Arn --output text 2>/dev/null || true +} diff --git a/ops/terraform/.terraform.lock.hcl b/ops/terraform/.terraform.lock.hcl new file mode 100644 index 000000000..e976e31c7 --- /dev/null +++ b/ops/terraform/.terraform.lock.hcl @@ -0,0 +1,47 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.50.0" + constraints = ">= 6.0.0" + hashes = [ + "h1:mNg4vBXXqbO0hY2jCxhOyKVrnjEO0viTG2EY4oAlWaQ=", + "zh:0072806bb262c6d86bc25b4a75750e469881144c14818afdba7b82db840e1588", + "zh:1ebc2dae335dad7a8b16a1985b69a63a14954282bb44fdba7d5103f77551ac7b", + "zh:2dab48fe8f3193b8216d578ac1e3674fa566435cc7dbce2953d55b72e31d0241", + "zh:2fc3d3029c2b7429472391ef339672e1fca8e6ff32c8a519bf3acedafa7e24fe", + "zh:38a36e64e7212f6cedac861ea4d449cce07131b3378de601bf9d49a99e000208", + "zh:3ac70758ed251ce78b7f541a5a79cc6fe56474412783ae1decef719bdd0f30bf", + "zh:4385d3903e685bddb2b8005b4eb7db89f030267d4d03c7d792d2f5e739cc874a", + "zh:4cce0760b87fbafd51f30faec2a737f4183b7c615f4a86557f7d3c893a610dc5", + "zh:4feaeed18694239b896c6415d9a1e5ef89e1da4f4ad60924aa0522adeb1f6599", + "zh:502fca2be1c95f443c3e67d0555601d1de65b4ca82d197c059e9c868360e3a0a", + "zh:57d037f6fdd045f2660909c3bdface9622d81165ce647479cba98d1f353c5eab", + "zh:5dc5a0b915c2ac5256d909458f5c8e40b35f78b3a36ea893c86624eaf6c54e37", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b84c87c58a320adbb2c74a4cad03ae5aac7f2eae21db26f00fdde98c8c4d4523", + "zh:c895f1d5cbcbeff77850ac99efd36bde0048d4e909b296882331b9b9ebf48cfa", + "zh:ead82831683619124597a1f170dd31e9b293e9cf22f558cb166d5e734fcd11e4", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.3.0" + constraints = ">= 4.0.0" + hashes = [ + "h1:j/BqLS2N2AScZyotd9nZpHdieJ7e5S8y+A+ZfIu8kL8=", + "zh:0ab58d6f8991d436c7d2dbd89ed814709b949b07ac5a54ee53b0aec1fa772a8b", + "zh:60b347abcb56f45d97c56f14d895069cd15a83993f199777f571b79fea3642ee", + "zh:6889be32640349230de3f23856e6f04e0e9ced4a84a27d3f552fa54684448218", + "zh:73f8e1ecf7135033165fb14b7e8bf4d656f3ce13065ec35762ea0481975328c7", + "zh:94ce25ee253eca0b42cae9c856b36bca8103b6453012d1b279c3623c805f2d42", + "zh:96bc6de9fd67bc446fd11257872e1ffb1029a996ed1d65a3f6b43f6d408ad9ab", + "zh:97c609a310a51bfd504d704e036d72064a84bf0bdb36cc08cd4cc66098212b41", + "zh:a12c16e94533c5bd123f75032576b9dc91dd5d5ccd5f7cf331d0f2e1adc55cf8", + "zh:c4f014f876adf7af57188795050bda5b0029d8c7d7773031102b6c36dcf1fc21", + "zh:d9b0a21583aaa3df3a95394fb949a3c515ff71c2ff5a1fc4a73d364aa90bfca5", + "zh:da510d22f0c6d71ad19a76406f106b782448f512375787ecfabb338ed1e311a7", + "zh:f0e9447a9ce3a24cdaa113089e65663c836d8b9bfdb915a1c0284e0112cab5c0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/ops/terraform/azure/.terraform.lock.hcl b/ops/terraform/azure/.terraform.lock.hcl new file mode 100644 index 000000000..20700bc7e --- /dev/null +++ b/ops/terraform/azure/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "3.8.0" + constraints = ">= 3.7.0" + hashes = [ + "h1:kIEmiknJFWKe44U9ePG5vXwoloKep0Dbi/mQ8uqCbw0=", + "zh:0d26cfbf9417acd1c2295ccd5b0052abeac85ad1c3f6422ff09bf6a1ce16f00d", + "zh:144d4ea92fed541a6376bc76ad65ba4738dfd7bcab4c9d6cc20d35001338d06d", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:2061d2cb64d8167d0af37e6610d0dc051977bd1ccc0e5cdd5ab02525ee239f96", + "zh:75562fdc3b313b7e538907199bfa588a1fcbc40113b0f3b7bfb496fbc358a32f", + "zh:78bef022ae9b1b0c636b7dd32bcda13fb273023f3a888cc005f3aa20e365b417", + "zh:ad8dbee59843154f8e93b24db9939a4257d13c7c86331eb93f1691294bc4e31f", + "zh:b3d83d7ac57073631704336a188cb746c473f728fc7ccb76abecb520e83fdf65", + "zh:c0bf9e0be73843de9089597be2720e4093b3ba320fbad99ab86da47681e77949", + "zh:c9a4c27d2b0800d3f4ece19d66c1fa574f7cd4ff66277af8f120d65e8f03f48e", + "zh:cd9ad8c848e17d9824045c33132cc0e87aa4d58cdb7bee6c0f6c3f9bc27892d5", + "zh:fbcafc21cdd19451274b905f9ee8a5b758ca63cc231e7544c815642e4a399c6d", + ] +} diff --git a/ops/terraform/azure/main.tf b/ops/terraform/azure/main.tf new file mode 100644 index 000000000..d36c2e2d3 --- /dev/null +++ b/ops/terraform/azure/main.tf @@ -0,0 +1,48 @@ +# Azure workload identity federation so Windows codesigning (Azure Trusted +# Signing via signtool dlib) authenticates with Buildkite OIDC tokens +# instead of an AZURE_CLIENT_SECRET. +# +# This is a separate root module from ../ because it authenticates against +# a different control plane (Microsoft Graph, not AWS); apply it with +# credentials that may manage the Trusted Signing app registration. +# +# Buildkite `sub` claims embed the commit sha, so exact-match federated +# credentials cannot work. Use flexible federated identity credentials +# (claims matching expressions) to wildcard-match protected refs of the +# julia-publish pipeline (Windows codesigning happens in the trusted +# publish step). Requires Microsoft Graph support for flexible FIC on the +# tenant; if unavailable, fall back to `--subject-claim organization_id` +# tokens (exact-match credential on the Buildkite organization UUID) at +# the cost of org-level granularity -- see ops/README.md. + +locals { + bk_oidc_host = "agent.buildkite.com" + + # Must stay in sync with publish_sub_patterns in ../iam.tf. + publish_sub_patterns = { + master = "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/master:*" + release = "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/release-*:*" + tags = "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/tags/v*:*" + } +} + +data "azuread_application" "trusted_signing" { + client_id = var.azure_app_id +} + +resource "azuread_application_flexible_federated_identity_credential" "buildkite" { + for_each = local.publish_sub_patterns + + application_id = data.azuread_application.trusted_signing.id + display_name = "buildkite-julia-publish-${each.key}" + description = "Buildkite julia-publish (${each.key}) Windows Trusted Signing" + audience = "api://AzureADTokenExchange" + issuer = "https://${local.bk_oidc_host}" + claims_matching_expression = "claims['sub'] matches '${each.value}'" +} + +# Publish jobs authenticate via: +# buildkite-agent oidc request-token --audience api://AzureADTokenExchange +# with AZURE_CLIENT_ID=, AZURE_TENANT_ID, and +# AZURE_FEDERATED_TOKEN_FILE set by utilities/windows/codesign.sh. +# Remove any AZURE_CLIENT_SECRET from the app once this works. diff --git a/ops/terraform/azure/variables.tf b/ops/terraform/azure/variables.tf new file mode 100644 index 000000000..8d903aca9 --- /dev/null +++ b/ops/terraform/azure/variables.tf @@ -0,0 +1,10 @@ +variable "azure_app_id" { + description = "Client ID of the existing app registration that holds the Trusted Signing 'Code Signing Certificate Profile Signer' role assignment" + type = string +} + +variable "bk_org" { + description = "Buildkite organization slug" + type = string + default = "julialang" +} diff --git a/ops/terraform/azure/versions.tf b/ops/terraform/azure/versions.tf new file mode 100644 index 000000000..b96c61266 --- /dev/null +++ b/ops/terraform/azure/versions.tf @@ -0,0 +1,13 @@ +terraform { + # azuread >= 3.7 is needed for + # azuread_application_flexible_federated_identity_credential. + required_version = ">= 1.5" + required_providers { + azuread = { + source = "hashicorp/azuread" + version = ">= 3.7" + } + } +} + +provider "azuread" {} diff --git a/ops/terraform/buildkite_ids.auto.tfvars b/ops/terraform/buildkite_ids.auto.tfvars new file mode 100644 index 000000000..0ef22b3a0 --- /dev/null +++ b/ops/terraform/buildkite_ids.auto.tfvars @@ -0,0 +1,31 @@ +# Buildkite UUIDs pinned by the IAM trust policies (see variables.tf). +# Not secrets; fetched 2026-06-12 from the Buildkite REST API. These are +# static -- refetch only if a pipeline is ever recreated or moved between +# clusters (and re-apply). +# +# julia-publish runs in its own cluster, separate from the build +# pipelines: the cluster_id condition then also stops a compromised +# build-pool agent from minting publish-shaped tokens. + +buildkite_organization_id = "d409823c-5fa7-41c8-9033-7269c5fde4f3" + +buildkite_pipeline_ids = { + "julia-pr" = "019ebd61-5b1f-428e-b08c-1b5a2111e001" + "julia-ci" = "019ebd63-635e-4155-b60a-9d2815900786" + "julia-publish" = "019ebd63-df36-4f53-a07f-4b31064df0f8" + "julia-buildkite-ci" = "019f3d33-d54f-4087-b70f-767d53b801d4" +} + +buildkite_cluster_ids = { + "julia-pr" = "ae7e6bd1-fde8-433d-bac7-9d2d01108ed6" + "julia-ci" = "ae7e6bd1-fde8-433d-bac7-9d2d01108ed6" + "julia-buildkite-ci" = "ae7e6bd1-fde8-433d-bac7-9d2d01108ed6" + "julia-publish" = "fd6c2af4-60c1-40ee-bdd5-88ecb6698fbc" +} + +# Isolated NON-PRODUCTION publish test stack (ops/terraform/test_publish.tf). +# Set to the real UUIDs to enable the throwaway test keys/bucket/role; comment +# out (or set null) to disable. julia-publish-test-nosecrets lives in the same +# Secure cluster as julia-publish. +buildkite_test_pipeline_id = "019ec73b-f8df-428a-a256-745eff852687" +buildkite_test_cluster_id = "fd6c2af4-60c1-40ee-bdd5-88ecb6698fbc" diff --git a/ops/terraform/iam.tf b/ops/terraform/iam.tf new file mode 100644 index 000000000..2e0452067 --- /dev/null +++ b/ops/terraform/iam.tf @@ -0,0 +1,392 @@ +# IAM roles assumed by Buildkite jobs via OIDC. +# +# Trust is split into an untrusted tier and a trusted tier so that pull +# request builds can never reach signing keys or release locations: +# +# The untrusted roles exist once per build pipeline -- julia-pr and +# julia-ci never share a role, so a role's identity IS its pipeline: +# +# julia-oidc-stage-pr UNTRUSTED. Assumable from any job of julia-pr +# julia-oidc-stage-ci (resp. julia-ci); the build step assumes it to +# stage its tarball directly. May only write unsigned +# artifacts, write-once, to a path gated by the build's +# own commit sha, inside that one pipeline's OWN +# ephemeral staging bucket (publish only ever reads +# julia-ci's). No KMS, no final-location write. Because +# the permissions are harmless, the spoofable `ref` +# component of the sub claim does not matter here. +# julia-oidc-stage-buildkite UNTRUSTED. Same shape for the +# julia-buildkite repository's own self-test pipeline +# (julia-buildkite-ci), which builds julia with +# proposed pipeline code: write-once staging to its +# own bucket, which nothing downstream consumes. +# julia-oidc-tokens-ci Telemetry bearer tokens from SSM; julia-ci ONLY. +# PR builds get no tokens whatsoever: a malicious pull +# request executes code inside the job and could read +# and exfiltrate any bearer token available to it. +# julia-oidc-publish TRUSTED. kms:Sign with the signing keys, read the +# julia-ci staging bucket, and write the final release +# locations. Assumable ONLY from the `julia-publish` +# pipeline slug. That pipeline must have pull-request +# builds DISABLED and be branch-limited to +# master/release-*/v* (see ops/README.md). Since a PR +# can never produce a build under that slug, the slug +# is the trust boundary -- not the (PR-spoofable) +# branch name. +# julia-oidc-docs-deploy TRUSTED. kms:Sign with the docs SSH key; publish +# pipeline only. +# +# Defense in depth: trusted publish jobs additionally run +# utilities/verify_trusted_commit.sh before assuming their role, which +# refuses unless BUILDKITE_COMMIT is reachable from a protected ref of the +# canonical JuliaLang/julia repository. +# +# Overwrite protection: every PutObject must use the S3 conditional write +# header (s3:if-none-match = "*"), i.e. uploads fail if the object already +# exists. The only exception is `julia-latest-*` pointer objects, which the +# publish pipeline intentionally repoints. + +data "aws_caller_identity" "current" {} + +locals { + # `sub` format: organization:ORG:pipeline:PIPELINE:ref:REF:commit:SHA:step:STEP + + # The UNTRUSTED build pipelines (short suffix used in role names): + # julia-pr builds pull requests + # julia-ci builds trusted refs (master / release-* / tags), incl. scheduled + # Any ref is allowed: the stage/token roles these map to are deliberately + # harmless, so we do not (and must not) rely on the ref component for trust. + build_pipelines = { + "julia-pr" = "pr" + "julia-ci" = "ci" + # the julia-buildkite repository's own self-test CI + "julia-buildkite-ci" = "buildkite" + } + + # The TRUSTED publish pipeline. julia-publish MUST be configured in + # Buildkite with pull-request builds disabled and branch-limited to the + # protected refs (see ops/README.md), and is triggered only by julia-ci. + # A PR can then never produce a build under this slug, so the slug itself + # is the trust boundary; the ref patterns below are belt-and-braces only. + publish_sub_patterns = [ + "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/master:*", + "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/release-*:*", + "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/tags/v*:*", + ] + + # Per-role trust: which pipeline (by slug pattern in `sub` AND by + # unforgeable UUID session tag) and -- for the trusted roles -- which + # steps (aws:RequestTag/step_key, attested by the agent) may assume it. + # Every role trusts exactly ONE pipeline. The untrusted roles have no + # step restriction (null): build steps stage artifacts directly, test + # steps post results directly, and the permissions are deliberately + # harmless anyway. + oidc_trust = { + stage-pr = { + pipelines = ["julia-pr"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-pr:*"] + step_key_patterns = null + } + stage-ci = { + pipelines = ["julia-ci"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-ci:*"] + step_key_patterns = null + } + stage-buildkite = { + pipelines = ["julia-buildkite-ci"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-buildkite-ci:*"] + step_key_patterns = null + } + # NB: there is deliberately no tokens-pr. A pull request executes + # attacker-controlled code inside the job, which could exfiltrate any + # bearer token the job can read -- so PR builds get NO tokens at all. + tokens-ci = { + pipelines = ["julia-ci"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-ci:*"] + step_key_patterns = null + } + publish = { + pipelines = ["julia-publish"] + sub_patterns = local.publish_sub_patterns + step_key_patterns = ["publish_*"] + } + docs-deploy = { + pipelines = ["julia-publish"] + sub_patterns = local.publish_sub_patterns + step_key_patterns = ["deploy_docs"] + } + } + + # Allowed cluster UUIDs per role, from the per-pipeline cluster map; + # roles whose pipelines have no configured cluster get no condition. + trust_cluster_ids = { + for role, t in local.oidc_trust : + role => distinct(compact([for p in t.pipelines : lookup(var.buildkite_cluster_ids, p, "")])) + } + + # Artifact-family prefixes inside each staging bucket (normal release / + # no-GPL; see S3_BUCKET_PREFIX selection in utilities/build_envs.sh) + # and the final release locations. + staging_prefixes = [ + var.s3_bucket_prefix, + var.s3_nogpl_prefix, + ] + staging_bucket_arns = { + for slug, bucket in var.s3_staging_buckets : slug => "arn:aws:s3:::${bucket}" + } + final_paths = [ + "arn:aws:s3:::${var.s3_bucket}/${var.s3_bucket_prefix}", + "arn:aws:s3:::${var.s3_nogpl_bucket}/${var.s3_nogpl_prefix}", + ] +} + +data "aws_iam_policy_document" "trust" { + for_each = local.oidc_trust + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity", "sts:TagSession"] + + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.buildkite.arn] + } + + condition { + test = "StringEquals" + variable = "${local.bk_oidc_host}:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringLike" + variable = "${local.bk_oidc_host}:sub" + values = each.value.sub_patterns + } + dynamic "condition" { + for_each = each.value.step_key_patterns != null ? [1] : [] + content { + test = "StringLike" + variable = "aws:RequestTag/step_key" + values = each.value.step_key_patterns + } + } + + # Pin the Buildkite UUIDs, not just the slugs in `sub`: slugs are + # renameable/re-mintable, UUIDs are not. These arrive as session tags + # (utilities/aws_oidc.sh) since IAM cannot condition on other claims. + # A token requested without these tags fails the conditions outright. + condition { + test = "StringEquals" + variable = "aws:RequestTag/organization_id" + values = [var.buildkite_organization_id] + } + condition { + test = "StringEquals" + variable = "aws:RequestTag/pipeline_id" + values = [for p in each.value.pipelines : var.buildkite_pipeline_ids[p]] + } + dynamic "condition" { + for_each = length(local.trust_cluster_ids[each.key]) > 0 ? [1] : [] + content { + test = "StringEquals" + variable = "aws:RequestTag/cluster_id" + values = local.trust_cluster_ids[each.key] + } + } + } +} + +# ---- julia-oidc-stage-{pr,ci} (UNTRUSTED, one role per build pipeline) ------ +# Each role is assumable from exactly one pipeline (trust policy) and can +# write-once to exactly that pipeline's staging bucket, only below the +# build's own commit sha (the build_commit tag is attested by Buildkite, +# not settable by the job). julia-pr's role never mentions the julia-ci +# bucket that publish reads from. + +resource "aws_iam_role" "stage" { + for_each = local.build_pipelines + + name = "julia-oidc-stage-${each.value}" + description = "Buildkite ${each.key} staging upload: write-once to ${var.s3_staging_buckets[each.key]}, own commit path (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["stage-${each.value}"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "stage" { + for_each = local.build_pipelines + + statement { + sid = "WriteOnceToOwnCommitStagingPath" + # No PutObjectAcl: the staging buckets disable ACLs (BucketOwnerEnforced, + # public read via bucket policy), and the s3:if-none-match condition key + # only exists for the PutObject action anyway. + actions = ["s3:PutObject"] + # $${...} is the literal IAM policy variable, not Terraform interpolation. + resources = [ + for p in local.staging_prefixes : + "${local.staging_bucket_arns[each.key]}/${p}/$${aws:PrincipalTag/build_commit}/*" + ] + + condition { + test = "StringEquals" + variable = "s3:if-none-match" + values = ["*"] + } + } + + statement { + sid = "ReadStagingForRetryChecks" + actions = ["s3:GetObject"] + resources = ["${local.staging_bucket_arns[each.key]}/*"] + } +} + +resource "aws_iam_role_policy" "stage" { + for_each = local.build_pipelines + + name = "write-once-uploads-to-own-commit-staging-path" + role = aws_iam_role.stage[each.key].id + policy = data.aws_iam_policy_document.stage[each.key].json +} + +# ---- julia-oidc-publish (TRUSTED) ------------------------------------------- +# Sign + promote staged artifacts to the final release locations. Assumable +# only from the julia-publish pipeline slug (PR builds disabled there). + +resource "aws_iam_role" "publish" { + name = "julia-oidc-publish" + description = "Buildkite publish: sign + promote staged artifacts to release (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["publish"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "publish" { + statement { + sid = "ReadStagedArtifacts" + actions = ["s3:GetObject"] + # Only the julia-ci staging bucket: artifacts staged by pull-request + # builds (julia-pr's bucket) are not publishable inputs. + resources = ["${local.staging_bucket_arns["julia-ci"]}/*"] + } + + statement { + sid = "WriteOnceToFinalLocations" + actions = ["s3:PutObject"] + resources = [for p in local.final_paths : "${p}/*"] + + condition { + test = "StringEquals" + variable = "s3:if-none-match" + values = ["*"] + } + } + + # Versioned objects are uploaded public-read (--acl). The s3:if-none-match + # condition key exists only for s3:PutObject, so a PutObjectAcl folded into + # the conditioned statement above would never match -- keep it unconditioned. + # Object content stays write-once: that is enforced by the PutObject grant. + statement { + sid = "SetPublicReadAclOnFinalObjects" + actions = ["s3:PutObjectAcl"] + resources = [for p in local.final_paths : "${p}/*"] + } + + statement { + sid = "RepointLatestPointers" + actions = ["s3:PutObject", "s3:PutObjectAcl"] + resources = flatten([for p in local.final_paths : [ + "${p}/*/julia-latest-*", + "${p}/*/*/julia-latest-*", + "${p}/*/*/*/julia-latest-*", + ]]) + } + + statement { + sid = "ReadFinalForRetryChecks" + actions = ["s3:GetObject"] + resources = [for p in local.final_paths : "${p}/*"] + } + + statement { + sid = "ReleaseSigning" + actions = ["kms:Sign", "kms:GetPublicKey"] + resources = [ + aws_kms_key.macos_codesign.arn, + aws_kms_external_key.notary_api.arn, + aws_kms_key.tarball_signing.arn, + ] + + condition { + test = "StringLike" + variable = "aws:PrincipalTag/step_key" + values = ["publish_*"] + } + } +} + +resource "aws_iam_role_policy" "publish" { + name = "sign-and-promote-staged-artifacts-to-release" + role = aws_iam_role.publish.id + policy = data.aws_iam_policy_document.publish.json +} + +# ---- julia-oidc-docs-deploy (TRUSTED) --------------------------------------- + +resource "aws_iam_role" "docs_deploy" { + name = "julia-oidc-docs-deploy" + description = "Buildkite docs deploy: SSH signing via KMS (aws-kms-pkcs11, via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["docs-deploy"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "docs_deploy" { + statement { + sid = "SshSignViaDocsDeployKey" + actions = ["kms:Sign", "kms:GetPublicKey"] + resources = [aws_kms_key.docs_deploy.arn] + + condition { + test = "StringEquals" + variable = "aws:PrincipalTag/step_key" + values = ["deploy_docs"] + } + } + # No S3 statement is needed to fetch the staged HTML docs: the staging bucket + # grants public s3:GetObject via its bucket policy, and deploy_docs.yml reads + # the htmldocs by its exact key (no ListBucket / --recursive). +} + +resource "aws_iam_role_policy" "docs_deploy" { + name = "ssh-sign-docs-deploy-via-kms" + role = aws_iam_role.docs_deploy.id + policy = data.aws_iam_policy_document.docs_deploy.json +} + +# ---- julia-oidc-tokens-ci ---------------------------------------------------- +# CI telemetry bearer tokens (codecov, coveralls, buildkite analytics) are +# inherently symmetric secrets, so they live in SSM Parameter Store +# (SecureString, see ops/23_put_tokens.sh) and are fetched at runtime by +# this role. Nothing secret is stored in the repository or this state. +# julia-ci ONLY -- pull request builds run attacker-controlled code and +# therefore get no bearer tokens at all (there is no tokens-pr role). + +resource "aws_iam_role" "tokens" { + name = "julia-oidc-tokens-ci" + description = "Buildkite julia-ci telemetry: read coverage/analytics tokens from SSM (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["tokens-ci"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "tokens" { + statement { + sid = "ReadTelemetryTokens" + actions = ["ssm:GetParameter"] + resources = ["arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm_token_prefix}/*"] + } +} + +resource "aws_iam_role_policy" "tokens" { + name = "read-telemetry-tokens-from-ssm" + role = aws_iam_role.tokens.id + policy = data.aws_iam_policy_document.tokens.json +} diff --git a/ops/terraform/kms.tf b/ops/terraform/kms.tf new file mode 100644 index 000000000..4ed12efe1 --- /dev/null +++ b/ops/terraform/kms.tf @@ -0,0 +1,101 @@ +# The KMS keys used by Julia CI. No private key ever exists outside KMS +# (or, for the BYOK notary key, outside the one-time trusted import +# machine). +# +# The EXTERNAL-origin notary key is created here *without* key material; +# import the existing .p8 out-of-band (it must never enter the Terraform +# state) with: +# +# ops/21_import_notary_key.sh -> alias/julia-notary-api +# +# Until then it sits in the PendingImport state and cannot sign. + +# Fresh key; a new Developer ID certificate must be issued for it +# (see ops/22_generate_macos_csr.sh). +resource "aws_kms_key" "macos_codesign" { + description = "Julia macOS Developer ID codesigning key (used via rcodesign)" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = "RSA_2048" + + # The signing keys are the only unrecoverable resources in this module + # (the GPG key exists nowhere else; the notary .p8 is destroyed after + # import; the others are bound to issued certificates / registered + # deploy keys). prevent_destroy makes any plan that would destroy or + # REPLACE one (key specs are immutable, so e.g. a key_spec edit plans + # as replacement) a hard error. 30 days is the maximum (and default) + # cancellation window for a scheduled deletion; set explicitly. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "macos_codesign" { + name = "alias/julia-macos-codesigning" + target_key_id = aws_kms_key.macos_codesign.key_id +} + +# The existing App Store Connect API .p8 key is imported (Apple generates +# ASC API keys; we cannot register our own public key with Apple). +resource "aws_kms_external_key" "notary_api" { + description = "Julia App Store Connect API key (notarization JWT signing)" + key_usage = "SIGN_VERIFY" + key_spec = "ECC_NIST_P256" + + # See the lifecycle note on aws_kms_key.macos_codesign. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "notary_api" { + name = "alias/julia-notary-api" + target_key_id = aws_kms_external_key.notary_api.id +} + +# Fresh key generated inside KMS; the OpenPGP public key for it is +# exported with ops/20_export_gpg_pubkey.py and must be published as the +# new Julia releases signing key (signatures do NOT verify against the +# pre-migration juliareleases.asc). +resource "aws_kms_key" "tarball_signing" { + description = "Julia release tarball GPG signing key" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = var.tarball_key_spec + + # See the lifecycle note on aws_kms_key.macos_codesign. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "tarball_signing" { + name = "alias/julia-tarball-signing" + target_key_id = aws_kms_key.tarball_signing.key_id +} + +# The SSH key used to push to docs.julialang.org. SSH signs via the +# aws-kms-pkcs11 provider, so the key never exists outside KMS. Register +# the public key (ops/24_docs_deploy_pubkey.sh) as a GitHub deploy key +# with write access. +resource "aws_kms_key" "docs_deploy" { + description = "Julia docs.julialang.org SSH deploy key (used via aws-kms-pkcs11)" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = "ECC_NIST_P256" + + # See the lifecycle note on aws_kms_key.macos_codesign. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "docs_deploy" { + name = "alias/julia-docs-deploy" + target_key_id = aws_kms_key.docs_deploy.key_id +} diff --git a/ops/terraform/oidc.tf b/ops/terraform/oidc.tf new file mode 100644 index 000000000..271a4fafc --- /dev/null +++ b/ops/terraform/oidc.tf @@ -0,0 +1,22 @@ +# IAM OIDC identity provider for Buildkite agents. +# +# Once this exists, Buildkite jobs can exchange `buildkite-agent oidc +# request-token --audience sts.amazonaws.com` tokens for temporary AWS +# credentials via sts:AssumeRoleWithWebIdentity (no static secrets). + +locals { + bk_oidc_host = "agent.buildkite.com" +} + +# AWS now validates most OIDC issuers against trusted root CAs and ignores +# the thumbprint, but the API still requires one: use the root certificate +# of the issuer's chain. +data "tls_certificate" "buildkite" { + url = "https://${local.bk_oidc_host}" +} + +resource "aws_iam_openid_connect_provider" "buildkite" { + url = "https://${local.bk_oidc_host}" + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = [data.tls_certificate.buildkite.certificates[0].sha1_fingerprint] +} diff --git a/ops/terraform/outputs.tf b/ops/terraform/outputs.tf new file mode 100644 index 000000000..f31d9b2de --- /dev/null +++ b/ops/terraform/outputs.tf @@ -0,0 +1,35 @@ +output "oidc_provider_arn" { + description = "IAM OIDC provider for agent.buildkite.com" + value = aws_iam_openid_connect_provider.buildkite.arn +} + +output "role_arns" { + description = "IAM roles assumed by Buildkite jobs" + value = { + stage-pr = aws_iam_role.stage["julia-pr"].arn + stage-ci = aws_iam_role.stage["julia-ci"].arn + tokens-ci = aws_iam_role.tokens.arn + publish = aws_iam_role.publish.arn + docs-deploy = aws_iam_role.docs_deploy.arn + } +} + +output "kms_key_arns" { + description = "KMS signing keys (the EXTERNAL notary key needs material imported)" + value = { + macos_codesign = aws_kms_key.macos_codesign.arn + notary_api = aws_kms_external_key.notary_api.arn + tarball_signing = aws_kms_key.tarball_signing.arn + docs_deploy = aws_kms_key.docs_deploy.arn + } +} + +output "staging_buckets" { + description = "Per-pipeline ephemeral staging buckets (PR consumers like juliaup read the julia-pr one)" + value = { for slug, b in aws_s3_bucket.staging : slug => b.bucket } +} + +output "julia_ci_aws_account_id" { + description = "Fill this into JULIA_CI_AWS_ACCOUNT_ID in utilities/aws_oidc.sh" + value = data.aws_caller_identity.current.account_id +} diff --git a/ops/terraform/s3.tf b/ops/terraform/s3.tf new file mode 100644 index 000000000..196a6e69a --- /dev/null +++ b/ops/terraform/s3.tf @@ -0,0 +1,73 @@ +# The per-pipeline ephemeral staging buckets. Everything in them is +# unsigned, write-once (enforced by IAM, see iam.tf), keyed by source +# commit sha, publicly readable (juliaup fetches PR binaries from the +# julia-pr bucket anonymously), and expired by lifecycle policy. +# +# The release buckets (julialangnightlies, julialang-nogpl) predate this +# module and are not managed here. + +resource "aws_s3_bucket" "staging" { + for_each = var.s3_staging_buckets + + bucket = each.value +} + +# Object ACLs are disabled entirely (the modern S3 arrangement): public +# readability comes from the bucket policy below, and uploads must NOT +# send --acl (an ACL'd PUT would both fail under BucketOwnerEnforced and +# require s3:PutObjectAcl, which the stage roles do not have -- the +# s3:if-none-match condition key only exists for the PutObject action). +resource "aws_s3_bucket_ownership_controls" "staging" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_public_access_block" "staging" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false +} + +# Everything staged is world-readable: PR binaries are consumed by juliaup +# anonymously, and the artifacts are public release candidates anyway. +resource "aws_s3_bucket_policy" "staging_public_read" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + depends_on = [aws_s3_bucket_public_access_block.staging] + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "PublicReadStagedArtifacts" + Effect = "Allow" + Principal = "*" + Action = "s3:GetObject" + Resource = "${each.value.arn}/*" + }] + }) +} + +resource "aws_s3_bucket_lifecycle_configuration" "staging" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + rule { + id = "expire-staged-artifacts" + status = "Enabled" + filter { + prefix = "" + } + expiration { + days = var.staging_expiry_days[each.key] + } + } +} diff --git a/ops/terraform/test_publish.tf b/ops/terraform/test_publish.tf new file mode 100644 index 000000000..501fa29b0 --- /dev/null +++ b/ops/terraform/test_publish.tf @@ -0,0 +1,242 @@ +# Isolated, NON-PRODUCTION publish test stack for `julia-publish-test-nosecrets`. +# +# This duplicates just enough of the real publish stack to exercise the full +# download -> KMS-sign -> promote flow end to end, but against THROWAWAY KMS +# keys (with self-signed certs) and a SEPARATE bucket -- so the publish flow +# can be debugged freely without touching any production key, bucket, or the +# public `julia-latest-*` pointers. +# +# Everything here is self-contained in this one file and gated on +# `var.buildkite_test_pipeline_id` being set: a normal `terraform apply` +# (test pipeline UUID unset) creates none of it. To tear the test stack down, +# delete this file (the keys have no prevent_destroy) -- production is untouched. +# +# Deliberately omitted (per design): Windows Authenticode signing (Azure) and +# macOS notarization (Apple ASC) have no non-production equivalent, so the test +# pipeline skips them (PUBLISH_SKIP_WINDOWS_SIGN / PUBLISH_SKIP_NOTARIZATION). +# Hence only two test keys: GPG tarball signing and macOS codesigning. + +variable "buildkite_test_pipeline_id" { + description = "UUID of the julia-publish-test-nosecrets Buildkite pipeline (set in buildkite_ids.auto.tfvars to enable the test stack)" + type = string + default = null + + validation { + condition = var.buildkite_test_pipeline_id == null ? true : can(regex("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.buildkite_test_pipeline_id)) && var.buildkite_test_pipeline_id != "00000000-0000-0000-0000-000000000000" + error_message = "buildkite_test_pipeline_id must be the pipeline's UUID (or null to disable the test stack)." + } +} + +variable "buildkite_test_cluster_id" { + description = "UUID of the cluster the julia-publish-test-nosecrets pipeline runs in (optional; omit for no cluster condition)" + type = string + default = null +} + +variable "s3_test_publish_bucket" { + description = "Bucket for the isolated publish test stack (staging input + promotion output)" + type = string + default = "julialang-test-publish" +} + +locals { + # The whole test stack materializes only once the pipeline UUID is supplied. + enable_test_publish = var.buildkite_test_pipeline_id != null + test_count = local.enable_test_publish ? 1 : 0 + # Cluster condition only if a test cluster UUID was given. + test_cluster_ids = compact([var.buildkite_test_cluster_id == null ? "" : var.buildkite_test_cluster_id]) +} + +# ---- Throwaway test signing keys (no prevent_destroy) ----------------------- +# Same specs as production (kms.tf): RSA so kms_gpg_sign.py's OpenPGP assembly +# works, RSA_2048 for the macOS codesign key. Public halves become self-signed +# material committed to the repo (ops/20_export_gpg_pubkey.py and +# ops/32_gen_test_codesign_cert.sh). + +resource "aws_kms_key" "tarball_signing_test" { + count = local.test_count + description = "TEST (non-production) Julia release tarball GPG signing key" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = var.tarball_key_spec + deletion_window_in_days = 7 +} + +resource "aws_kms_alias" "tarball_signing_test" { + count = local.test_count + name = "alias/julia-tarball-signing-test" + target_key_id = aws_kms_key.tarball_signing_test[0].key_id +} + +resource "aws_kms_key" "macos_codesign_test" { + count = local.test_count + description = "TEST (non-production) Julia macOS codesigning key (self-signed cert, used via rcodesign)" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = "RSA_2048" + deletion_window_in_days = 7 +} + +resource "aws_kms_alias" "macos_codesign_test" { + count = local.test_count + name = "alias/julia-macos-codesigning-test" + target_key_id = aws_kms_key.macos_codesign_test[0].key_id +} + +# ---- Test bucket (staging input + promotion output) ------------------------- +# One bucket used for both: seeded staging at bin//... and promotion at +# bin///... (distinct key shapes). Mirrors the ephemeral-bucket setup +# (s3.tf): ACLs off, public-read via policy, short lifecycle expiry. + +resource "aws_s3_bucket" "test_publish" { + count = local.test_count + bucket = var.s3_test_publish_bucket +} + +resource "aws_s3_bucket_ownership_controls" "test_publish" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_public_access_block" "test_publish" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false +} + +resource "aws_s3_bucket_policy" "test_publish_public_read" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + depends_on = [aws_s3_bucket_public_access_block.test_publish] + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "PublicReadTestArtifacts" + Effect = "Allow" + Principal = "*" + Action = "s3:GetObject" + Resource = "${aws_s3_bucket.test_publish[0].arn}/*" + }] + }) +} + +resource "aws_s3_bucket_lifecycle_configuration" "test_publish" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + rule { + id = "expire-test-artifacts" + status = "Enabled" + filter { + prefix = "" + } + expiration { + days = 7 + } + } +} + +# ---- Test OIDC role + standalone trust -------------------------------------- +# Trust is intentionally LOOSE on ref (any ref of the test pipeline slug) so we +# can test from any branch -- but still pinned to the unforgeable org_id + test +# pipeline_id (+ optional cluster_id) session tags, so ONLY the real test +# pipeline can assume it. The role can sign with the test keys and read/write +# ONLY the test bucket. No write-once condition, so the same commit can be +# re-published repeatedly while debugging. + +data "aws_iam_policy_document" "publish_test_trust" { + count = local.test_count + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity", "sts:TagSession"] + + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.buildkite.arn] + } + + condition { + test = "StringEquals" + variable = "${local.bk_oidc_host}:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringLike" + variable = "${local.bk_oidc_host}:sub" + values = ["organization:${var.bk_org}:pipeline:julia-publish-test-nosecrets:*"] + } + condition { + test = "StringEquals" + variable = "aws:RequestTag/organization_id" + values = [var.buildkite_organization_id] + } + condition { + test = "StringEquals" + variable = "aws:RequestTag/pipeline_id" + values = [var.buildkite_test_pipeline_id] + } + dynamic "condition" { + for_each = length(local.test_cluster_ids) > 0 ? [1] : [] + content { + test = "StringEquals" + variable = "aws:RequestTag/cluster_id" + values = local.test_cluster_ids + } + } + } +} + +resource "aws_iam_role" "publish_test" { + count = local.test_count + name = "julia-oidc-publish-test" + description = "NON-PRODUCTION publish test: sign with test KMS keys + read/write the test bucket (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.publish_test_trust[0].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "publish_test" { + count = local.test_count + + # Read seeded staging input AND read-back for retry checks; write promoted + # artifacts. No s3:if-none-match: overwrites are allowed so the same build + # can be re-run freely. Scoped to the test bucket only. + statement { + sid = "ReadTestBucket" + actions = ["s3:GetObject"] + resources = ["${aws_s3_bucket.test_publish[0].arn}/*"] + } + statement { + sid = "WriteTestBucket" + actions = ["s3:PutObject"] + resources = ["${aws_s3_bucket.test_publish[0].arn}/*"] + } + + statement { + sid = "TestReleaseSigning" + # DescribeKey: kms_gpg_sign.py --public-key-from-kms reads the GPG key's + # CreationDate (part of the OpenPGP fingerprint) so the test pipeline need + # not commit a pubkey. GetPublicKey: fetch the RSA public half. + actions = ["kms:Sign", "kms:GetPublicKey", "kms:DescribeKey"] + resources = [ + aws_kms_key.macos_codesign_test[0].arn, + aws_kms_key.tarball_signing_test[0].arn, + ] + } +} + +resource "aws_iam_role_policy" "publish_test" { + count = local.test_count + name = "test-sign-and-promote-to-test-bucket" + role = aws_iam_role.publish_test[0].id + policy = data.aws_iam_policy_document.publish_test[0].json +} + +output "test_publish_role_arn" { + description = "ARN of the non-production publish test role (null until the test pipeline UUID is set)" + value = local.enable_test_publish ? aws_iam_role.publish_test[0].arn : null +} diff --git a/ops/terraform/variables.tf b/ops/terraform/variables.tf new file mode 100644 index 000000000..3c20443f0 --- /dev/null +++ b/ops/terraform/variables.tf @@ -0,0 +1,142 @@ +variable "aws_region" { + description = "Region holding the KMS keys and SSM parameters" + type = string + default = "us-east-1" +} + +variable "bk_org" { + description = "Buildkite organization slug" + type = string + default = "julialang" +} + +# The Buildkite UUIDs pinned by the IAM trust policies, alongside the +# slug-based `sub` patterns. Slugs are renameable and can be re-minted by +# deleting + recreating a pipeline; UUIDs cannot, so a recreated pipeline +# with a matching slug does not regain role access. The jobs pass these as +# AWS session tags (see utilities/aws_oidc.sh) because IAM can only +# condition on aud/sub from the raw token. Values live in +# buildkite_ids.auto.tfvars (fetched once from the Buildkite REST API: +# GET /v2/organizations/ and /v2/organizations//pipelines/; +# re-fetch only if a pipeline is ever recreated or moved between clusters). +# No defaults and nullable = false: an apply without real, well-formed +# UUIDs (set in buildkite_ids.auto.tfvars) must be impossible -- a trust +# policy missing these conditions would fall back to slug-only pinning. +variable "buildkite_organization_id" { + description = "UUID of the Buildkite organization" + type = string + nullable = false + + validation { + # The null guard keeps `terraform validate` (which leaves required + # variables unset) happy; nullable = false rejects null at plan time. + condition = var.buildkite_organization_id == null ? true : can(regex("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.buildkite_organization_id)) && var.buildkite_organization_id != "00000000-0000-0000-0000-000000000000" + error_message = "Set buildkite_organization_id to the organization UUID (see buildkite_ids.auto.tfvars)." + } +} + +variable "buildkite_pipeline_ids" { + description = "UUIDs of the four Buildkite pipelines, keyed by slug" + type = map(string) + nullable = false + + validation { + condition = var.buildkite_pipeline_ids == null ? true : keys(var.buildkite_pipeline_ids) == tolist(["julia-buildkite-ci", "julia-ci", "julia-pr", "julia-publish"]) + error_message = "buildkite_pipeline_ids must have exactly the keys julia-buildkite-ci, julia-ci, julia-pr, julia-publish." + } + validation { + condition = var.buildkite_pipeline_ids == null ? true : alltrue([ + for id in values(var.buildkite_pipeline_ids) : + can(regex("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", id)) && id != "00000000-0000-0000-0000-000000000000" + ]) + error_message = "Set every pipeline UUID (see buildkite_ids.auto.tfvars)." + } +} + +# Cluster UUID each pipeline runs in, keyed by pipeline slug. Optional: +# pipelines without an entry get no cluster condition (e.g. unclustered +# agents, where the cluster_id claim is empty). +variable "buildkite_cluster_ids" { + description = "UUIDs of the Buildkite cluster each pipeline belongs to, keyed by pipeline slug" + type = map(string) + default = {} +} + +variable "s3_bucket" { + description = "S3 bucket that release binaries are uploaded to" + type = string + default = "julialangnightlies" +} + +variable "s3_bucket_prefix" { + description = "Prefix under s3_bucket for release binaries" + type = string + default = "bin" +} + +# Bucket + prefix for the scheduled no-GPL builds +# (see pipelines/scheduled/platforms/upload_*.no_gpl.yml) +variable "s3_nogpl_bucket" { + description = "S3 bucket for the scheduled no-GPL builds" + type = string + default = "julialang-nogpl" +} + +variable "s3_nogpl_prefix" { + description = "Prefix under s3_nogpl_bucket" + type = string + default = "bin-nogpl" +} + +# Per-pipeline EPHEMERAL staging buckets (created by this module, with +# lifecycle expiry). The build step of each untrusted pipeline writes its +# unsigned artifacts write-once to ///... +# julia-pr and julia-ci get SEPARATE buckets: the trusted publish pipeline +# only reads the julia-ci bucket, so a pull-request build can never place +# (or, since paths are write-once, pre-claim) anything publish would +# consume. PR consumers (juliaup) read from the julia-pr bucket. +variable "s3_staging_buckets" { + description = "Ephemeral staging bucket per untrusted pipeline, keyed by pipeline slug" + type = map(string) + default = { + "julia-pr" = "julialang-ephemeral-pr" + "julia-ci" = "julialang-ephemeral-ci" + # The julia-buildkite repo's own self-test CI: nothing consumes these + # artifacts (the builds only prove the pipeline code works), and the + # bucket separation means a self-test build can never place anything + # juliaup or julia-publish would read. + "julia-buildkite-ci" = "julialang-ephemeral-buildkite" + } +} + +variable "staging_expiry_days" { + description = "Days before staged objects expire, keyed by pipeline slug" + type = map(number) + default = { + # PR binaries are consumed by juliaup while the PR is open + "julia-pr" = 90 + # publish promotes within hours of staging + "julia-ci" = 30 + # self-test artifacts have no consumers; keep them briefly for humans + "julia-buildkite-ci" = 14 + } +} + +# RSA matching the strength of the previous (pre-KMS) release signing key. +# Must be an RSA spec: the OpenPGP signature assembly in +# utilities/kms_gpg_sign.py only supports RSA. +variable "tarball_key_spec" { + description = "KMS key spec for the GPG release tarball signing key" + type = string + default = "RSA_4096" +} + +# Bearer tokens (codecov, coveralls, buildkite analytics) are inherently +# symmetric secrets, so they live in SSM Parameter Store. The parameter +# *values* are deliberately not managed here (they would end up in the +# Terraform state); store them with ops/23_put_tokens.sh. +variable "ssm_token_prefix" { + description = "SSM Parameter Store prefix for CI telemetry bearer tokens" + type = string + default = "/julia-ci/tokens" +} diff --git a/ops/terraform/versions.tf b/ops/terraform/versions.tf new file mode 100644 index 000000000..caf2fef80 --- /dev/null +++ b/ops/terraform/versions.tf @@ -0,0 +1,24 @@ +terraform { + # aws >= 6.0 is needed for asymmetric (SIGN_VERIFY) key specs on + # aws_kms_external_key (the BYOK notary key). + required_version = ">= 1.5" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + } +} + +provider "aws" { + region = var.aws_region + default_tags { + tags = { + project = "julia-ci" + } + } +} diff --git a/pipelines/main/0_webui.yml b/pipelines/main/0_webui.yml index 8ef688a20..1fc3dffa0 100644 --- a/pipelines/main/0_webui.yml +++ b/pipelines/main/0_webui.yml @@ -1,30 +1,51 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +# This file records the steps configured in the Buildkite webUI; modifying +# this file has no effect -- paste it into the webUI when it changes. +# +# These steps are pasted VERBATIM into BOTH untrusted build pipelines: +# +# julia-pr -- builds pull requests, and nothing else. Settings: +# * Trigger builds after pushing code: OFF +# * Build pull requests: ON (including from third-party +# forked repositories) +# julia-ci -- builds trusted refs only. Settings: +# * Trigger builds after pushing code: ON, +# branch limit: master release-* +# * Build tags: ON (v*) +# * Build pull requests: OFF +# * the nightly build schedule lives here +# +# Keeping the two configurations identical is deliberate: the only +# difference in behavior is downstream, where the rendered julia-publish +# trigger is gated on `pipeline.slug == "julia-ci"`. Neither pipeline +# holds any secrets or signing rights: jobs obtain credentials at runtime +# via Buildkite OIDC (see utilities/aws_oidc.sh), and AWS IAM decides what +# each job identity may do (PR builds: write-once staging uploads only). +# The trusted signing/promotion steps run in julia-publish (see +# pipelines/publish/0_webui.yml), never here. +# +# The same steps are ALSO pasted into a third untrusted pipeline: +# +# julia-buildkite-ci -- self-test CI for the julia-buildkite repository. +# Its pipeline repository is JuliaCI/julia-buildkite (not +# JuliaLang/julia), so its jobs run the repository hook +# .buildkite/hooks/post-checkout, which swaps in a julia +# checkout and pins the external-buildkite meta-data to the +# julia-buildkite commit under test. Settings: +# * Trigger builds after pushing code: ON, branch limit: main +# * Build pull requests: ON (including from third-party +# forked repositories) +# See .buildkite/README.md. steps: - - group: "Upload" + - group: "Launch" steps: - - label: "Unlock secrets, launch pipelines" + - label: "Launch pipelines" agents: queue: "launch" - cryptic_capable: "true" os: "linux" plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/main/launch_unsigned_builders.yml - - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/main/launch_signed_jobs.yml - signature_file: .buildkite/pipelines/main/launch_signed_jobs.yml.signature - command: "true" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/main/launch_untrusted_builders.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/launch_coverage.yml diff --git a/pipelines/main/README.md b/pipelines/main/README.md index 6b9d67bd7..900ca1f32 100644 --- a/pipelines/main/README.md +++ b/pipelines/main/README.md @@ -1,15 +1,22 @@ ## Main pipeline -This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests. +This directory contains most of the builders. They are triggered by GitHub +webhook events (pushes and pull requests). -We have a different main pipeline for each permanent branch. +Builds are split across three Buildkite pipelines by trust level (see +`ops/README.md` for why): -For example: +| Pipeline | Builds | Trust | +| --------------- | ------------------------------------------------- | ------------------------------ | +| `julia-pr` | pull requests | untrusted (stage only) | +| `julia-ci` | `master`, `release-*`, tags, and scheduled nightlies | untrusted to sign; triggers publish | +| `julia-publish` | (triggered by `julia-ci`) signs + promotes | trusted (KMS signing keys) | -| Permanent Branch | Pipeline | -| ---------------- | -------------------------------------------------------------------------------- | -| `master` | [`julia-master`](https://buildkite.com/julialang/julia-master) | -| `release-1.6` | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) | -| `release-1.7` | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) | - -(This is not a complete list.) +Each build step stages its unsigned tarball directly (write-once, no relay +jobs) to a commit-sha-gated path in its pipeline's own ephemeral staging +bucket: `julia-pr` builds go to `julialang-ephemeral-pr` (where juliaup +finds PR binaries) and stop there. Trusted-ref builds run in `julia-ci`, +stage to `julialang-ephemeral-ci`, and trigger `julia-publish`, which +signs and promotes — reading only the `julia-ci` bucket. `julia-publish` +does not build pull requests, so a PR can never reach the signing keys or +feed artifacts into publishing. diff --git a/pipelines/main/launch_coverage.yml b/pipelines/main/launch_coverage.yml new file mode 100644 index 000000000..675e0c8a5 --- /dev/null +++ b/pipelines/main/launch_coverage.yml @@ -0,0 +1,26 @@ +# Launch the coverage pipeline. +# +# Coverage is launched here (a SEPARATE pipeline upload from the build/test +# launchers) so that it is NOT behind the publish barrier: a long scheduled +# coverage run must never delay the julia-publish trigger. The wait+trigger +# live at the end of launch_untrusted_builders.yml instead, inlined with the +# build/test groups they gate. +steps: + - group: "Upload" + steps: + - label: "Launch coverage" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Launch the code coverage job (scheduled julia-ci builds and + # "needs full CI"-labeled PRs; PR builds hold no coverage tokens, + # so they collect coverage without uploading it -- see the + # conditionals inside) + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/coverage/coverage.yml + + agents: + queue: "build" + os: "linux" + arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml deleted file mode 100644 index e3abfaff2..000000000 --- a/pipelines/main/launch_signed_jobs.yml +++ /dev/null @@ -1,42 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: "Launch signed jobs" - command: "true" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - verbose: true - signed_pipelines: - # Launch the upload jobs - - pipeline: .buildkite/pipelines/main/launch_upload_jobs.yml - signature_file: .buildkite/pipelines/main/launch_upload_jobs.yml.signature - inputs: - - .buildkite/pipelines/main/launch_upload_jobs.yml - - .buildkite/utilities/arches_pipeline_upload.sh - - .buildkite/utilities/arches_env.sh - - .buildkite/utilities/build_envs.sh - - .buildkite/utilities/extract_triplet.sh - - .buildkite/utilities/upload_julia.sh - - .buildkite/utilities/aws_config - - .buildkite/utilities/sign_tarball.sh - - .buildkite/utilities/macos/build_dmg.sh - - .buildkite/utilities/macos/codesign.sh - - .buildkite/pipelines/main/platforms/launch_powerpc.jl - - .buildkite/pipelines/main/platforms/upload_linux.arches - - .buildkite/pipelines/main/platforms/upload_linux.yml - - .buildkite/pipelines/main/platforms/upload_macos.arches - - .buildkite/pipelines/main/platforms/upload_macos.yml - - .buildkite/pipelines/main/platforms/upload_windows.arches - - .buildkite/pipelines/main/platforms/upload_windows.yml - - .buildkite/pipelines/main/misc/upload_buildkite_results.yml - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature deleted file mode 100644 index aab63da12..000000000 --- a/pipelines/main/launch_signed_jobs.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__ KX@OxCkd?] H\_#*?3A"pz#F]Eq*=b!9eF @LJkn cN8lN \ No newline at end of file diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml deleted file mode 120000 index 31a037322..000000000 --- a/pipelines/main/launch_unsigned_builders.yml +++ /dev/null @@ -1 +0,0 @@ -launch_unsigned_jobs.yml \ No newline at end of file diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml deleted file mode 100644 index b8ef15cd3..000000000 --- a/pipelines/main/launch_unsigned_jobs.yml +++ /dev/null @@ -1,213 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> launch_unsigned_builders.yml -> misc/llvmpasses.yml -# -# when we could theoretically just have the `webui` launch `misc/llvmpasses.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. - -steps: - - group: "Build" - notify: - - github_commit_status: - context: "Build" - steps: - - label: "Launch build jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 - # A few notes: - # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. - # This is intentional. We want to make sure that this script doesn't suddenly - # break when a new version of the plugin is released. - # 2. We specify the Julia version up to patch, e.g. '1.7.3'. - # This is intentional. We don't do e.g. '1.7', because we want to make sure - # that this script doesn't suddenly break when a new patch release of Julia - # is released. - version: '1.10.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - commands: | - ### Launch Linux build jobs. - # Regular: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Build" \ - ALLOW_FAIL="false" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - ### Launch macOS build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - ### Launch FreeBSD build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_freebsd.arches \ - .buildkite/pipelines/main/platforms/build_freebsd.yml - ### Launch Windows build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_windows.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - group: "Check" - notify: - - github_commit_status: - context: "Check" - steps: - - label: "Launch check jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite - - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - group: "Test" - notify: - - github_commit_status: - context: "Test" - steps: - - label: "Launch test jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - export ALLOW_FAIL="false" - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml - - ### Launch Linux test jobs. - # Regular: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # i686-linux-gnu: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.i686.arches \ - .buildkite/pipelines/main/platforms/test_linux.i686.yml - ### Launch macOS test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml - echo "+++ Finished launching test jobs" - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - group: "Allow Fail" - steps: - - label: "Launch allowed-to-fail build jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail build jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS allowed-to-fail build jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - label: "Launch allowed-to-fail test jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 - # A few notes: - # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. - # This is intentional. We want to make sure that this script doesn't suddenly - # break when a new version of the plugin is released. - # 2. We specify the Julia version up to patch, e.g. '1.7.3'. - # This is intentional. We don't do e.g. '1.7', because we want to make sure - # that this script doesn't suddenly break when a new patch release of Julia - # is released. - version: '1.10.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - commands: | - export ALLOW_FAIL="true" - - ### Launch Linux allowed-to-fail test jobs. - # Regular: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Allow Fail" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/test_linux.powerpc.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - ### Launch macOS allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD allowed-to-fail jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/main/launch_untrusted_builders.yml b/pipelines/main/launch_untrusted_builders.yml new file mode 100644 index 000000000..66633640a --- /dev/null +++ b/pipelines/main/launch_untrusted_builders.yml @@ -0,0 +1,69 @@ +# This file launches all the UNTRUSTED build + test jobs -- which in the +# post-cryptic setup is every job of the julia-pr / julia-ci pipelines: +# none of them hold secrets or signing rights (the trusted work happens in +# the separate julia-publish Buildkite pipeline). +# +# LAUNCH-LATENCY OPTIMIZATION: +# +# Previously this file had five separate launch steps (groups Build / Check / +# Test / Allow Fail) that each ran MANY `buildkite-agent pipeline upload` +# calls -- roughly ~30 in total, one per arches-templated platform YAML and +# per static misc YAML. Each upload is a ~1.5s network round-trip, and they +# ran serially, so they dominated CI launch latency. +# +# Now a SINGLE launch step runs `utilities/render_launch_pipeline.py`, which +# renders ONE pre-grouped pipeline document (one `group:` per label: +# Build / Check / Test / Allow Fail / JuliaSyntax / JuliaC, plus the +# wait + julia-publish trigger), and uploads it with ONE `pipeline upload`. +# +# The renderer reproduces the old behaviour EXACTLY: +# * The arches-templated platform YAMLs (platforms/{build,test}_*.yml) are +# rendered once per arch, with the same per-arch variable substitution the +# old per-file `pipeline upload` performed (GROUP / ALLOW_FAIL / arches +# columns). Job grouping, keys, depends_on and soft_fail are preserved. +# * The static / launcher misc YAMLs (analyzegc, gcext, juliac, juliasyntax, +# ...) are emitted VERBATIM -- their double-dollar runtime escapes and their +# launch-env vars (e.g. gcext/test_revise, whose soft_fail is keyed off +# ALLOW_FAIL) are left for THIS single `pipeline upload` to interpolate. +# That is why we still `export ALLOW_FAIL="false"` below. +# * PowerPC is omitted: launch_powerpc.jl only uploaded powerpc arches for +# Julia < 1.12, and is a no-op on current master (1.14). +# +# Yes, this is creating another layer of indirection; the flow now looks like: +# +# [webui] -> launch_untrusted_builders.yml -> render_launch_pipeline.py +# +# however this raises the bar for contributors to add new steps to our CI +# configuration, so I'd rather live with an extra layer of indirection and +# only need to touch the webui configuration when something about the +# launch flow itself changes. + +steps: + - label: ":rocket: Launch jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # gcext.yml / test_revise.yml are emitted verbatim by the renderer and + # still have a soft_fail keyed off ALLOW_FAIL; this single `pipeline + # upload` resolves it -- exactly as the old Test launch step did. + export ALLOW_FAIL="false" + python3 .buildkite/utilities/render_launch_pipeline.py > pipeline.generated.yml + buildkite-agent pipeline upload pipeline.generated.yml + agents: + queue: "launch" + os: "linux" + arch: "x86_64" # prevent from running on PiBots + + # The barrier + trigger of the trusted julia-publish pipeline are emitted by + # render_launch_pipeline.py at the END of the rendered document, in the SAME + # `pipeline upload` as the Build/Check/Test/Allow-Fail groups. That is on + # purpose so the `wait` reliably barriers all of those dynamically-uploaded + # jobs (a `wait` placed in a separately-uploaded file used to race the build + # jobs and fire the trigger before anything was built or staged). + # + # See render_launch_pipeline.py for the wait + the `julia-publish` trigger + # (guarded by `if: pipeline.slug == "julia-ci"`). Coverage is launched by a + # separate upload (launch_coverage.yml) and is intentionally NOT behind this + # barrier. diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml deleted file mode 100644 index 22c1ae236..000000000 --- a/pipelines/main/launch_upload_jobs.yml +++ /dev/null @@ -1,103 +0,0 @@ -# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 - -steps: - - group: "Upload" - steps: - - label: "Launch upload jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 - # A few notes: - # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. - # This is intentional. We want to make sure that this script doesn't suddenly - # break when a new version of the plugin is released. - # 2. We specify the Julia version up to patch, e.g. '1.7.3'. - # This is intentional. We don't do e.g. '1.7', because we want to make sure - # that this script doesn't suddenly break when a new patch release of Julia - # is released. - version: '1.10.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - export ALLOW_FAIL="false" - - ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done - ### Linux: - # Regular Linux upload jobs: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - # PowerPC Linux upload jobs, which we only run for Julia prior to 1.12: - GROUP="Upload" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/upload_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - ### macOS: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_macos.arches \ - .buildkite/pipelines/main/platforms/upload_macos.yml - ### FreeBSD: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_freebsd.arches \ - .buildkite/pipelines/main/platforms/upload_freebsd.yml - ### Windows: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_windows.arches \ - .buildkite/pipelines/main/platforms/upload_windows.yml - - ### Upload test results.json files to buildkite - for OS in linux linux.i686 macos freebsd windows; do \ - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_$${OS}.arches \ - .buildkite/pipelines/main/misc/upload_buildkite_results.yml - done - - # Launch doctest deploy job - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml - - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - - group: "Allow Fail (Upload)" - steps: - - label: "Launch allowed-to-fail upload jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - - export ALLOW_FAIL="true" - - # Launch Linux allowed-to-fail upload jobs - GROUP="Allow Fail (Upload)" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature deleted file mode 100644 index 786b715e6..000000000 --- a/pipelines/main/launch_upload_jobs.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__`3-leyw=lІZ+gsf9Leb"@pv4NSe|NwkQE "XUO; \ No newline at end of file diff --git a/pipelines/main/misc/analyzegc.yml b/pipelines/main/misc/analyzegc.yml index 60d5ae66f..c940d7b27 100644 --- a/pipelines/main/misc/analyzegc.yml +++ b/pipelines/main/misc/analyzegc.yml @@ -11,9 +11,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/llvm_passes.x86_64.tar.gz - rootfs_treehash: "0badf5d8794e21a03ac2ff2d46c1ab538ac02571" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/llvm_passes.x86_64.tar.gz + rootfs_treehash: "630126d5595f428ef824651b9be020ffc485eb6f" workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" diff --git a/pipelines/main/misc/deploy_docs.yml b/pipelines/main/misc/deploy_docs.yml index b0545105e..4466d715a 100644 --- a/pipelines/main/misc/deploy_docs.yml +++ b/pipelines/main/misc/deploy_docs.yml @@ -2,14 +2,16 @@ steps: - group: "Upload" steps: - label: "upload docs" - depends_on: - - "doctest" - # We only upload if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: ((build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/)) || (pipeline.slug == "julia-buildkite") + # The step key IS this job's OIDC identity: julia-oidc-docs-deploy only + # trusts step_key "deploy_docs", from the trusted julia-publish pipeline + # (this file is launched by the publish pipeline, not the build one). + # The key MUST be on this job -- the one that actually assumes the role + # -- not on the launcher step in publish/launch.yml that uploads this + # file (that launcher touches no AWS). There is no `depends_on` doctest + # here: the html docs tarball is read from the commit-sha gated S3 + # staging area (the build pipeline's doctest step stages it), the same + # cross-pipeline handoff used for the binaries. + key: "deploy_docs" # Prevent multiple pipelines from pushing to the `git@github.com:JuliaLang/docs.julialang.org` repo simultaneously concurrency: 1 concurrency_group: 'upload/julialang/upload_docs/docs_julialang_org' @@ -23,23 +25,38 @@ steps: persist_depot_dirs: packages,artifacts,compiled # We maintain multiple Documenter manifests (one manifest for each v1.x minor version of Julia). version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - # Use `aws_uploader` image, since it has things like `git` and `ssh` and whatnot - rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/aws_uploader.x86_64.tar.gz" - rootfs_treehash: "61cb30497604eb1581b3ffd6a8058c150f4cbf8b" + - JuliaCI/sandbox#v2: + # Use the julia_publish image (same trusted toolchain as the + # publish_all step): it has git/ssh/awscli AND the aws-kms-pkcs11 + # PKCS#11 provider used to SSH-sign the docs push with the KMS + # deploy key. (Keep this treehash in sync with publish/launch.yml's + # -- it is the same image.) + rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v8.6/julia_publish.x86_64.tar.gz" + rootfs_treehash: "8b28c5b8937f25827c1ad4b6a3bc1bb90c9e9087" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" - - JuliaCI/cryptic#v2: - files: - - .buildkite/secrets/ssh_docs_deploy - - staticfloat/ssh-agent#v1: - keyfiles: - - .buildkite/secrets/ssh_docs_deploy commands: | - buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" . + echo "--- Download built docs from staging" + # The build pipeline's doctest step stages the html docs under the + # commit-sha-gated key julia-htmldocs.tar.gz (see misc/doctest.yml). + # We fetch it by exact key with a single GetObject -- the staging + # bucket grants public s3:GetObject via its bucket policy, so no IAM + # S3 grant (and no bucket-listing) is needed here. We still assume the + # docs-deploy role below for the KMS-backed SSH signing of the push. + source .buildkite/utilities/aws_oidc.sh docs-deploy + aws s3 cp "s3://${STAGING_BUCKET:-julialang-ephemeral-ci}/${S3_BUCKET_PREFIX:-bin}/${BUILDKITE_COMMIT}/julia-htmldocs.tar.gz" . + + echo "--- Configure SSH signing via AWS KMS" + # The deploy key private half lives in AWS KMS and never leaves + # it: ssh performs each signature through the aws-kms-pkcs11 + # provider, authorized by this job's OIDC identity (the docs-deploy + # role assumed above). No secrets are involved. + mkdir -p ~/.config/aws-kms-pkcs11 + echo '{"slots": [{"kms_key_id": "alias/julia-docs-deploy"}]}' > ~/.config/aws-kms-pkcs11/config.json + export GIT_SSH_COMMAND="ssh -o PKCS11Provider=$${AWS_KMS_PKCS11_PROVIDER:-/usr/lib/x86_64-linux-gnu/pkcs11/aws_kms_pkcs11.so}" echo "--- clone docs.julialang.org" mkdir -p /home/juliaci/.ssh @@ -48,13 +65,13 @@ steps: cd repo echo "--- deploy new docs" - # Delete everything except for `.git` - shopt -s extglob - rm -rf !(".git") - shopt -u extglob + # Delete everything except for `.git` (find, not an extglob `!(.git)` + # glob, which fails to parse unless `shopt -s extglob` is already + # active when the commands block is parsed). + find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + # Extract new docs build - tar -zxf ../julia-*-htmldocs.tar.gz + tar -zxf ../julia-htmldocs.tar.gz # Build commit message echo "build based on $${BUILDKITE_COMMIT}" > ../commit_message @@ -68,19 +85,10 @@ steps: git add . git commit -a -F ../commit_message - if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then - # If we're testing new CI configurations, push to a temporary branch, then delete it - git push origin gh-pages:gh-pages-buildkite-$${BUILDKITE_COMMIT} - git push origin --delete gh-pages-buildkite-$${BUILDKITE_COMMIT} - else - git push origin gh-pages:gh-pages - fi + git push origin gh-pages:gh-pages timeout_in_minutes: 45 agents: - queue: "test" + queue: "default" sandbox_capable: "true" os: "linux" arch: "x86_64" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml index b4e9164a4..dbad83574 100644 --- a/pipelines/main/misc/doctest.yml +++ b/pipelines/main/misc/doctest.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: @@ -41,6 +41,38 @@ steps: echo "--- Compress and upload HTML docs" buildkite-agent artifact upload "$${DOCUMENTER_ARCHIVE}" + + echo "--- Stage HTML docs to S3 for the publish pipeline" + # Mirror the binary handoff (build_julia.sh): a write-once upload to + # this build's commit-sha-gated staging path via the untrusted + # `stage` role, which can do nothing else. The trusted julia-publish + # docs-deploy phase reads it back from the same path (see + # misc/deploy_docs.yml). STAGING_BUCKET selection mirrors + # build_envs.sh; docs are not platform/no-GPL specific, so the prefix + # is the default `bin`. + cd .. + # Documenter only produces the archive when it decides to deploy, + # which it refuses to do for pull-request builds -- on those there + # is nothing to stage (and no publish pipeline consumes it). + if [[ ! -f "doc/$${DOCUMENTER_ARCHIVE}" ]]; then + echo "No $${DOCUMENTER_ARCHIVE} produced (pull-request build); skipping staging." + exit 0 + fi + if [[ "$${BUILDKITE_PIPELINE_SLUG:-}" == "julia-pr" ]]; then + STAGING_BUCKET="julialang-ephemeral-pr" + elif [[ "$${BUILDKITE_PIPELINE_SLUG:-}" == julia-buildkite* ]]; then + STAGING_BUCKET="julialang-ephemeral-buildkite" + else + STAGING_BUCKET="julialang-ephemeral-ci" + fi + source .buildkite/utilities/aws_oidc.sh stage + source .buildkite/utilities/upload_to_s3.sh + # Stage under a stable, version-independent key so the publish + # pipeline's docs-deploy job can fetch it by exact key (a single + # public GetObject -- the staging bucket is public-read) rather than + # listing the bucket, which would need an S3 ListBucket grant. + UPLOAD_TO_S3_ACL=none upload_to_s3 "doc/$${DOCUMENTER_ARCHIVE}" \ + "$${STAGING_BUCKET}/$${S3_BUCKET_PREFIX:-bin}/$${BUILDKITE_COMMIT?}/julia-htmldocs.tar.gz" env: # Give a fake documenter key to get Documenter to try and deploy for us DOCUMENTER_KEY: "" diff --git a/pipelines/main/misc/embedding.yml b/pipelines/main/misc/embedding.yml index 7b9fd7933..d8c850579 100644 --- a/pipelines/main/misc/embedding.yml +++ b/pipelines/main/misc/embedding.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: diff --git a/pipelines/main/misc/gcext.yml b/pipelines/main/misc/gcext.yml index 4e58bd686..aabbe8f46 100644 --- a/pipelines/main/misc/gcext.yml +++ b/pipelines/main/misc/gcext.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: diff --git a/pipelines/main/misc/juliac/test_juliac.yml b/pipelines/main/misc/juliac/test_juliac.yml new file mode 100644 index 000000000..658e13031 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac.yml @@ -0,0 +1,32 @@ +steps: + - group: "JuliaC" + notify: + - github_commit_status: + context: "JuliaC" + steps: + - label: "Launch JuliaC test jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + export ALLOW_FAIL="${ALLOW_FAIL:-false}" + ### Launch Linux JuliaC test jobs: + GROUP="JuliaC" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/juliac/test_juliac_linux.arches \ + .buildkite/pipelines/main/misc/juliac/test_juliac_linux.yml + ### Launch macOS JuliaC test jobs: + GROUP="JuliaC" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/juliac/test_juliac_macos.arches \ + .buildkite/pipelines/main/misc/juliac/test_juliac_macos.yml + ### Launch Windows JuliaC test jobs: + GROUP="JuliaC" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/juliac/test_juliac_windows.arches \ + .buildkite/pipelines/main/misc/juliac/test_juliac_windows.yml + agents: + queue: "launch" + os: "linux" + arch: "x86_64" diff --git a/pipelines/main/misc/juliac/test_juliac_linux.arches b/pipelines/main/misc/juliac/test_juliac_linux.arches new file mode 100644 index 000000000..08ea47320 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_linux.arches @@ -0,0 +1,9 @@ +# ROOTFS_IMAGE_NAME TRIPLET ARCH ARCH_ROOTFS TIMEOUT ROOTFS_TAG ROOTFS_HASH ALLOW_FAIL +package_linux x86_64-linux-gnu x86_64 x86_64 . v7.3 d8de9e0051c5cd69f3dde8ddaefc594d0a71043b . +package_linux aarch64-linux-gnu aarch64 aarch64 . v7.3 d39fb11781df9b03e448ad9318df2d195a3acf26 true + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 +#default ALLOW_FAIL false diff --git a/pipelines/main/misc/juliac/test_juliac_linux.yml b/pipelines/main/misc/juliac/test_juliac_linux.yml new file mode 100644 index 000000000..4a3092e72 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_linux.yml @@ -0,0 +1,43 @@ +steps: + - group: "${GROUP?}" + steps: + - label: ":linux: test JuliaC ${TRIPLET?}" + key: "test-juliac-${TRIPLET?}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + persist_depot_dirs: packages,artifacts,compiled + version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_${TRIPLET}" 'julia-*-linux-*.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-linux-*.tar.gz + ln -s ./usr/bin/julia ./julia + + echo "--- Install and test JuliaC" + unset JULIA_DEPOT_PATH + if [ -f deps/jlutilities/juliac/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.activate("deps/jlutilities/juliac"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="JuliaC", rev="main"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + fi + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + agents: + queue: "test" + sandbox_capable: "true" + os: "linux" + arch: "${ARCH?}" diff --git a/pipelines/main/misc/juliac/test_juliac_macos.arches b/pipelines/main/misc/juliac/test_juliac_macos.arches new file mode 100644 index 000000000..f20eb7875 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_macos.arches @@ -0,0 +1,8 @@ +# OS TRIPLET ARCH TIMEOUT +macos x86_64-apple-darwin x86_64 . +macos aarch64-apple-darwin aarch64 . + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 diff --git a/pipelines/main/misc/juliac/test_juliac_macos.yml b/pipelines/main/misc/juliac/test_juliac_macos.yml new file mode 100644 index 000000000..5b8801625 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_macos.yml @@ -0,0 +1,34 @@ +steps: + - group: "${GROUP?}" + steps: + - label: ":macos: test JuliaC ${TRIPLET?}" + key: "test-juliac-${TRIPLET?}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_${TRIPLET}" 'julia-*-macos-*.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-macos-*.tar.gz + ln -s ./usr/bin/julia ./julia + + # Re-sign after download + .buildkite/utilities/macos/codesign.sh ./usr + + echo "--- Install and test JuliaC" + unset JULIA_DEPOT_PATH + if [ -f deps/jlutilities/juliac/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.activate("deps/jlutilities/juliac"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="JuliaC", rev="main"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + fi + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + agents: + queue: "test" + os: "macos" + arch: "${ARCH}" diff --git a/pipelines/main/misc/juliac/test_juliac_windows.arches b/pipelines/main/misc/juliac/test_juliac_windows.arches new file mode 100644 index 000000000..b56299780 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_windows.arches @@ -0,0 +1,7 @@ +# OS TRIPLET ARCH TIMEOUT +windows x86_64-w64-mingw32 x86_64 . + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 diff --git a/pipelines/main/misc/juliac/test_juliac_windows.yml b/pipelines/main/misc/juliac/test_juliac_windows.yml new file mode 100644 index 000000000..37980bd5d --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_windows.yml @@ -0,0 +1,29 @@ +steps: + - group: "${GROUP?}" + steps: + - label: ":windows: test JuliaC ${TRIPLET?}" + key: "test-juliac-${TRIPLET?}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_${TRIPLET}" 'julia-*-windows-*.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-windows-*.tar.gz + echo "--- Install and test JuliaC" + unset JULIA_DEPOT_PATH + if [ -f deps/jlutilities/juliac/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./usr/bin/julia.exe -e 'using Pkg; Pkg.activate("deps/jlutilities/juliac"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./usr/bin/julia.exe -e 'using Pkg; Pkg.add(name="JuliaC", rev="main"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + fi + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + agents: + queue: "test" + os: "windows" + arch: "${ARCH}" diff --git a/pipelines/main/misc/juliasyntax.launch.yml b/pipelines/main/misc/juliasyntax.launch.yml new file mode 100644 index 000000000..c892521e1 --- /dev/null +++ b/pipelines/main/misc/juliasyntax.launch.yml @@ -0,0 +1,31 @@ +# This is its own YAML file so that we can accomplish the following goal: +# If JuliaSyntax has not yet been moved into the JuliaLang/julia repo, +# then PRs to JuliaLang/julia should not display a `JuliaSyntax` PR commit status. + +steps: + - group: "JuliaSyntax" + notify: + - github_commit_status: + context: "JuliaSyntax" + steps: + - label: "Launch JuliaSyntax jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 + # A few notes: + # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. + # This is intentional. We want to make sure that this script doesn't suddenly + # break when a new version of the plugin is released. + # 2. We specify the Julia version up to patch, e.g. '1.7.3'. + # This is intentional. We don't do e.g. '1.7', because we want to make sure + # that this script doesn't suddenly break when a new patch release of Julia + # is released. + version: '1.10.10' + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/juliasyntax.test.yml + agents: + queue: "launch" + os: "linux" + arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/main/misc/juliasyntax.test.yml b/pipelines/main/misc/juliasyntax.test.yml new file mode 100644 index 000000000..9eb39efc5 --- /dev/null +++ b/pipelines/main/misc/juliasyntax.test.yml @@ -0,0 +1,215 @@ +steps: + - group: "JuliaSyntax" + steps: + # Note: This file only concerns testing JuliaSyntax on older Julia versions + # Testing JuliaSyntax on Julia#master is done separately. + # Specifically, for that case, testing JuliaSyntax is built into the regular `Base.runtests()`, + # so those tests are running in the regular Buildkite platform-specific test jobs. + + # This step has most of the JuliaSyntax test jobs + - label: "juliasyntax {{matrix.agent_os}}/{{matrix.julia_arch}}/{{matrix.julia_version}}" + # We comment out the Buildkite step key for this step, because: + # > You can't use matrix values in other attributes, including step keys and ... + # Source: https://buildkite.com/docs/pipelines/configure/workflows/build-matrix + # key: "juliasyntax-step-key" + timeout_in_minutes: 30 + agents: + queue: "test" + os: "{{matrix.agent_os}}" + arch: "{{matrix.agent_arch}}" + matrix: + setup: + agent_os: + - "linux" + - "windows" + # - "macos" # We handle macOS special, see "adjustments" section below + agent_arch: # arch of the Buildkite agent. Distinct from the Julia binary's arch + - "x86_64" # The "x86_64" agent arch is used for both 32-bit Julia and 64-bit Julia + julia_arch: # arch of the Julia binary to install. Distinct from the agent arch + - "x86_64" # 64-bit Julia + # 32-bit Julia on Linux is handled in a separate step later in this YAML file + # 32-bit Julia on Windows is handled in the "adjustments" section below + julia_version: + # Don't put "1" here. Always include at least major and minor version. + - "1.0" + - "1.6" # old LTS + - "1.10" # current LTS + - "1.12" # latest stable + adjustments: + # macOS x86_64 (Intel): + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.0" + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.6" # old LTS + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.10" # current LTS + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.12" # latest stable + # macOS aarch64 (Apple Silicon): + # Note: Julia 1.0 and 1.6 did not have native aarch64 binaries for macOS, so we skip those + - with: + agent_os: "macos" + agent_arch: "aarch64" + julia_arch: "aarch64" + julia_version: "1.10" # old LTS + - with: + agent_os: "macos" + agent_arch: "aarch64" + julia_arch: "aarch64" + julia_version: "1.12" # latest stable + # For some reason, the Julia 1.0 tarball for Windows x86_64 doesn't seem to exist. Weird. Skip it for now, try to fix this in the future. + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.0" + skip: true + # 32-bit Julia on Windows + # For some reason, the Julia 1.0 tarball for Windows i686 doesn't seem to exist. Weird. Skip it for now, try to fix this in the future. + # - with: + # agent_os: "windows" + # agent_arch: "x86_64" + # julia_arch: "i686" # 32-bit Julia + # julia_version: "1.0" + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "i686" # 32-bit Julia + julia_version: "1.6" # old LTS + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "i686" # 32-bit Julia + julia_version: "1.10" # current LTS + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "i686" # 32-bit Julia + julia_version: "1.12" # latest stable + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + isolated_depot: true + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '{{matrix.julia_version}}' + arch: '{{matrix.julia_arch}}' + commands: | + echo "--- Julia versioninfo" + julia -e 'import InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Pkg.dev-ing (by path) the local copy of JuliaSyntax" + export JULIA_PKG_PRECOMPILE_AUTO=0 + # Double dollar signs $$ needed here because of Buildkite stuff + echo "Old JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + # On Windows, the separator is ; + # On other platforms, the separator is : + sep=$$(julia -e 'println(Sys.iswindows() ? ";" : ":")') + # We change the depot path from `depot` to `depot:` + # The trailing colon/semicolon makes sure we can re-use the existing stdlib pkgimages, + # instead of needing to re-precompile them + # https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH + export JULIA_DEPOT_PATH="$${JULIA_DEPOT_PATH:?}$${sep}" + echo "New JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + julia -e 'import Pkg; pspec = Pkg.PackageSpec(path = "./JuliaSyntax"); Pkg.develop(pspec)' + + echo "--- Testing JuliaSyntax on Julia {{matrix.julia_version}}" + unset JULIA_PKG_PRECOMPILE_AUTO + julia -e 'import Pkg; Pkg.test("JuliaSyntax")' + + # This step just has the 32-bit Linux jobs for JuliaSyntax. + # For these jobs, we have to run inside a different rootfs image, because the default 64-bit agent rootfs image will complain + # when we try to run a 32-bit executable. + # The logic for the sandbox plugin is annoying, so I've separated it out. + # AFAICT, there is currently no way to conditionally skip a plugin. + # > Plugins run during the job lifecycle, before the conditional is evaluated + # Source: https://buildkite.com/docs/pipelines/configure/conditionals + - label: "juliasyntax {{matrix.agent_os}}/{{matrix.julia_arch}}/{{matrix.julia_version}} in i686 sandbox" + # We comment out the Buildkite step key for this step, because: + # > You can't use matrix values in other attributes, including step keys and ... + # Source: https://buildkite.com/docs/pipelines/configure/workflows/build-matrix + # key: "juliasyntax-step-key" + timeout_in_minutes: 30 + agents: + queue: "test" + os: "{{matrix.agent_os}}" + arch: "{{matrix.agent_arch}}" + matrix: + setup: + agent_os: + - "linux" + agent_arch: # arch of the Buildkite agent. Distinct from the Julia binary's arch + - "x86_64" # The "x86_64" agent arch is used for both 32-bit Julia and 64-bit Julia + julia_arch: # arch of the Julia binary to install. Distinct from the agent arch + - "i686" # 32-bit Julia + julia_version: + # Don't put "1" here. Always include at least major and minor version. + - "1.0" + - "1.6" # old LTS + - "1.10" # current LTS + - "1.12" # latest stable + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # This is the Julia install *outside* the sandbox. + # We only use this Julia install to run Sandbox.jl + # This is not the Julia install that will be used to run Pkg.test() + isolated_depot: true + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # current LTS. It doesn't really matter what version we pick here. + # arch: '{{matrix.julia_arch}}' # no need to specify the julia_arch for this "outside" Julia. + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.0/agent_linux.i686.tar.gz + rootfs_treehash: "cba648ddb9ac0d292f9c8809652523806674a3d2" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + - JuliaCI/julia#v1: + # This is the Julia install *inside* the sandbox. + # That's the Julia install we actually use for Pkg.test() + isolated_depot: true + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '{{matrix.julia_version}}' + arch: '{{matrix.julia_arch}}' + commands: | + echo "--- Julia versioninfo" + julia -e 'import InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Pkg.dev-ing (by path) the local copy of JuliaSyntax" + export JULIA_PKG_PRECOMPILE_AUTO=0 + # Double dollar signs $$ needed here because of Buildkite stuff + echo "Old JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + # On Windows, the separator is ; + # On other platforms, the separator is : + sep=$$(julia -e 'println(Sys.iswindows() ? ";" : ":")') + # We change the depot path from `depot` to `depot:` + # The trailing colon/semicolon makes sure we can re-use the existing stdlib pkgimages, + # instead of needing to re-precompile them + # https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH + export JULIA_DEPOT_PATH="$${JULIA_DEPOT_PATH:?}$${sep}" + echo "New JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + julia -e 'import Pkg; pspec = Pkg.PackageSpec(path = "./JuliaSyntax"); Pkg.develop(pspec)' + + echo "--- Testing JuliaSyntax on Julia {{matrix.julia_version}}" + unset JULIA_PKG_PRECOMPILE_AUTO + julia -e 'import Pkg; Pkg.test("JuliaSyntax")' diff --git a/pipelines/main/misc/llvmpasses.yml b/pipelines/main/misc/llvmpasses.yml index 89876105a..0684e0635 100644 --- a/pipelines/main/misc/llvmpasses.yml +++ b/pipelines/main/misc/llvmpasses.yml @@ -11,9 +11,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: diff --git a/pipelines/main/misc/pdf_docs/build_pdf_docs.yml b/pipelines/main/misc/pdf_docs/build_pdf_docs.yml index 4c0f657b2..17d672186 100644 --- a/pipelines/main/misc/pdf_docs/build_pdf_docs.yml +++ b/pipelines/main/misc/pdf_docs/build_pdf_docs.yml @@ -13,7 +13,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/latex.x86_64.tar.gz rootfs_treehash: "48f7c99f62ac7f2a8212d106059be1dddc3aa6bf" uid: 1000 diff --git a/pipelines/main/misc/sanitizers/asan.yml b/pipelines/main/misc/sanitizers/asan.yml index 9f377ad6a..f3df74443 100644 --- a/pipelines/main/misc/sanitizers/asan.yml +++ b/pipelines/main/misc/sanitizers/asan.yml @@ -11,16 +11,16 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/llvm_passes.x86_64.tar.gz - rootfs_treehash: "0badf5d8794e21a03ac2ff2d46c1ab538ac02571" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/llvm_passes.x86_64.tar.gz + rootfs_treehash: "630126d5595f428ef824651b9be020ffc485eb6f" uid: 1000 gid: 1000 workspaces: - "/cache/repos:/cache/repos" timeout_in_minutes: 180 if: | # We only run the `asan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + (build.branch != "release-1.6") && (build.branch != "release-1.7") commands: | echo "--- Build julia with ASAN" VERBOSE=1 contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} diff --git a/pipelines/main/misc/sanitizers/tsan.yml b/pipelines/main/misc/sanitizers/tsan.yml index 35743227c..e6dbc6b39 100644 --- a/pipelines/main/misc/sanitizers/tsan.yml +++ b/pipelines/main/misc/sanitizers/tsan.yml @@ -8,16 +8,16 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/llvm_passes.x86_64.tar.gz - rootfs_treehash: "0badf5d8794e21a03ac2ff2d46c1ab538ac02571" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/llvm_passes.x86_64.tar.gz + rootfs_treehash: "630126d5595f428ef824651b9be020ffc485eb6f" uid: 1000 gid: 1000 workspaces: - "/cache/repos:/cache/repos" timeout_in_minutes: 120 if: | # We only run the `tsan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + (build.branch != "release-1.6") && (build.branch != "release-1.7") commands: | echo "--- Build julia-debug runtime with TSAN" VERBOSE=1 contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug diff --git a/pipelines/main/misc/test_revise.yml b/pipelines/main/misc/test_revise.yml index 8705ad8da..e511c4036 100644 --- a/pipelines/main/misc/test_revise.yml +++ b/pipelines/main/misc/test_revise.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: @@ -30,7 +30,11 @@ steps: echo "--- Install and test Revise" unset JULIA_DEPOT_PATH - JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="Revise", rev="master"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("Revise")' + if [ -f deps/jlutilities/revise/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.activate("deps/jlutilities/revise"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("Revise")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="Revise", rev="master"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("Revise")' + fi timeout_in_minutes: 30 soft_fail: ${ALLOW_FAIL?} agents: diff --git a/pipelines/main/misc/trimming.yml b/pipelines/main/misc/trimming.yml index 514beb778..5890ae04a 100644 --- a/pipelines/main/misc/trimming.yml +++ b/pipelines/main/misc/trimming.yml @@ -14,15 +14,15 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" if: | # We only run the `trimming` job on Julia 1.12 and later. - (pipeline.slug != "julia-release-1-dot-10") && (pipeline.slug != "julia-release-1-dot-11") + (build.branch != "release-1.10") && (build.branch != "release-1.11") commands: | # Download pre-built julia, extract into `usr/` buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . diff --git a/pipelines/main/misc/upload_buildkite_results.yml b/pipelines/main/misc/upload_buildkite_results.yml deleted file mode 100644 index 0e5bd3759..000000000 --- a/pipelines/main/misc/upload_buildkite_results.yml +++ /dev/null @@ -1,76 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: "upload test results ${TRIPLET?}${USE_RR-}${i686_LABEL-}" - key: "upload_results_${TRIPLET?}${USE_RR-}${i686_LABEL-}" - depends_on: - # Wait for the builder to finish - - step: "build_${TRIPLET?}" - # Wait for the tester to finish - - step: "test_${TRIPLET?}${USE_RR-}${i686_LABEL-}" - allow_failure: true - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - BUILDKITE_ANALYTICS_TOKEN="U2FsdGVkX18BUSpNWNimRUOAh2ku0ugS7x3RRy1CfJBCxTLWArJpa1PXKoILfI7q" - timeout_in_minutes: ${TIMEOUT?} - soft_fail: true - commands: | - echo "--- Downloading test results from test step" - if ! buildkite-agent artifact download --step "test_${TRIPLET?}${USE_RR-}${i686_LABEL-}" "results.tar.gz" .; then - echo "No test results found from test_${TRIPLET?}${USE_RR-}${i686_LABEL-}" - echo "This could mean:" - echo " - The test step hasn't completed successfully yet" - echo " - The test step failed and produced no results" - echo " - This upload job was triggered manually before test completion" - echo "You can re-run this job manually once test results are available." - exit 1 - fi - - # Extract with relaxed permissions handling - if ! tar -xzf results.tar.gz --no-same-owner 2>/dev/null; then - echo "Failed to extract results.tar.gz, trying alternative extraction..." - if ! tar -xzf results.tar.gz --no-same-permissions --no-same-owner 2>/dev/null; then - echo "Could not extract results archive" - exit 1 - fi - fi - - echo "--- Uploading test results and assigning them to the test run" - export JOB_ID="$$(buildkite-agent meta-data get BUILDKITE_TEST_JOB_ID_test_${TRIPLET?}${USE_RR-}${i686_LABEL-} || echo "unknown")" - echo "Test job ID: $$JOB_ID" - - shopt -s nullglob # Ensure empty glob does not return a literal string - for file in results*.json; do - echo "Uploading $$file..." - # We can't use the test-collector plugin because it doesn't provide a way to set a custom `job_id` - curl \ - -X POST \ - --silent \ - --show-error \ - --max-time "${TIMEOUT?}" \ - -H "Authorization: Token token=\"$$BUILDKITE_ANALYTICS_TOKEN\"" \ - -F "data=@\"$$file\"" \ - -F "format=json" \ - -F "run_env[CI]=buildkite" \ - -F "run_env[key]=\"$$BUILDKITE_BUILD_ID\"" \ - -F "run_env[url]=\"$$BUILDKITE_BUILD_URL\"" \ - -F "run_env[branch]=\"$$BUILDKITE_BRANCH\"" \ - -F "run_env[commit_sha]=\"$$BUILDKITE_COMMIT\"" \ - -F "run_env[number]=\"$$BUILDKITE_BUILD_NUMBER\"" \ - -F "run_env[job_id]=\"$$JOB_ID\"" \ - -F "run_env[message]=\"$$BUILDKITE_MESSAGE\"" \ - https://analytics-api.buildkite.com/v1/uploads - echo "" - done - agents: - queue: "test" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - - diff --git a/pipelines/main/misc/whitespace.yml b/pipelines/main/misc/whitespace.yml index 56feb1aea..5c1e1a2d1 100644 --- a/pipelines/main/misc/whitespace.yml +++ b/pipelines/main/misc/whitespace.yml @@ -14,9 +14,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" workspaces: - "/cache/repos:/cache/repos" - JuliaCI/julia#v1: diff --git a/pipelines/main/platforms/build_freebsd.yml b/pipelines/main/platforms/build_freebsd.yml index 5ae3a8a33..76c459d04 100644 --- a/pipelines/main/platforms/build_freebsd.yml +++ b/pipelines/main/platforms/build_freebsd.yml @@ -4,6 +4,8 @@ steps: - label: ":freebsd: build ${TRIPLET?}" key: "build_${TRIPLET?}" plugins: + - JuliaCI/coreupload#v2: + core_pattern: "**/*.core" - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" @@ -19,3 +21,4 @@ steps: # Receive some environment mappings from our templating engine TRIPLET: "${TRIPLET?}" MAKE_FLAGS: "${MAKE_FLAGS?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml index 815034b40..9f8f19901 100644 --- a/pipelines/main/platforms/build_linux.yml +++ b/pipelines/main/platforms/build_linux.yml @@ -4,6 +4,10 @@ steps: - label: ":linux: build ${TRIPLET?}" key: "build_${TRIPLET?}" plugins: + - JuliaCI/coreupload#v2: + core_pattern: "**/*.core" + compressor: "zstd" + create_bundle: "true" - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" @@ -11,7 +15,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 @@ -34,3 +38,4 @@ steps: TRIPLET: "${TRIPLET?}" MAKE_FLAGS: "${MAKE_FLAGS?}" ROOTFS_IMAGE_NAME: "${ROOTFS_IMAGE_NAME?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml index 6e9583b34..16d2ce4cd 100644 --- a/pipelines/main/platforms/build_macos.yml +++ b/pipelines/main/platforms/build_macos.yml @@ -4,6 +4,10 @@ steps: - label: ":macos: build ${TRIPLET?}" key: "build_${TRIPLET?}" plugins: + - JuliaCI/coreupload#v2: + core_pattern: "**/*.core" + compressor: "zstd" + create_bundle: "true" - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" @@ -18,3 +22,4 @@ steps: # Receive some environment mappings from our templating engine TRIPLET: "${TRIPLET?}" MAKE_FLAGS: "${MAKE_FLAGS?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_freebsd.yml b/pipelines/main/platforms/test_freebsd.yml index 29b0c40ab..b73fd792f 100644 --- a/pipelines/main/platforms/test_freebsd.yml +++ b/pipelines/main/platforms/test_freebsd.yml @@ -24,3 +24,4 @@ steps: env: JULIA_SHELL: "/usr/local/bin/bash" TRIPLET: "${TRIPLET?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_linux.i686.yml b/pipelines/main/platforms/test_linux.i686.yml index 46b84f624..2cd39ce1b 100644 --- a/pipelines/main/platforms/test_linux.i686.yml +++ b/pipelines/main/platforms/test_linux.i686.yml @@ -20,7 +20,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 @@ -42,3 +42,4 @@ steps: TRIPLET: "${TRIPLET?}" USE_RR: "${USE_RR?}" i686_GROUP: "${i686_GROUP?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml index b69d51c02..5e2a487bc 100644 --- a/pipelines/main/platforms/test_linux.yml +++ b/pipelines/main/platforms/test_linux.yml @@ -20,7 +20,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 @@ -41,3 +41,4 @@ steps: JULIA_SHELL: "/bin/bash" TRIPLET: "${TRIPLET?}" USE_RR: "${USE_RR?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_macos.yml b/pipelines/main/platforms/test_macos.yml index 9cb4d8c97..620f4bacc 100644 --- a/pipelines/main/platforms/test_macos.yml +++ b/pipelines/main/platforms/test_macos.yml @@ -25,3 +25,4 @@ steps: env: JULIA_SHELL: "/bin/bash" TRIPLET: "${TRIPLET?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/upload_freebsd.yml b/pipelines/main/platforms/upload_freebsd.yml deleted file mode 100644 index 082e3bb51..000000000 --- a/pipelines/main/platforms/upload_freebsd.yml +++ /dev/null @@ -1,39 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: ":freebsd: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - files: - - .buildkite/secrets/tarball_signing.gpg - timeout_in_minutes: ${TIMEOUT?} - retry: - automatic: - - exit_status: "*" - limit: 3 - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - os: "freebsd" - arch: "${ARCH}" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml deleted file mode 100644 index abbe59468..000000000 --- a/pipelines/main/platforms/upload_linux.yml +++ /dev/null @@ -1,50 +0,0 @@ -steps: - - group: "${GROUP?}" - steps: - - label: ":linux: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/aws_uploader.x86_64.tar.gz - rootfs_treehash: "61cb30497604eb1581b3ffd6a8058c150f4cbf8b" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - files: - - .buildkite/secrets/tarball_signing.gpg - timeout_in_minutes: ${TIMEOUT?} - soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox_capable: "true" - os: "linux" - arch: "x86_64" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_macos.yml b/pipelines/main/platforms/upload_macos.yml deleted file mode 100644 index c186d27f9..000000000 --- a/pipelines/main/platforms/upload_macos.yml +++ /dev/null @@ -1,42 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: ":macos: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - NOTARIZATION_APPLE_ID="U2FsdGVkX1/b2eyuYolwIaW0DI2YE5tPcMYJ7X7/J5Qz6T/oV6bZEuQqUEet6Jvx" - - NOTARIZATION_APPLE_KEY="U2FsdGVkX1+z0XUSEnj0vLawcT1/U2wx1YDN6s3V9U5+qhbGwRQuXHNJXPmlNe/o" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/macos_codesigning.keychain - timeout_in_minutes: ${TIMEOUT?} - retry: - automatic: - - exit_status: "*" - limit: 3 - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - os: "macos" - arch: "${ARCH}" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_windows.yml b/pipelines/main/platforms/upload_windows.yml deleted file mode 100644 index 67a354b57..000000000 --- a/pipelines/main/platforms/upload_windows.yml +++ /dev/null @@ -1,62 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: ":windows: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw==" - - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ==" - - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA==" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/windows_codesigning.pfx - - docker#v3.13.0: - # We always use `x86_64` here, as it doesn't matter what the compiler is for uploading - image: "juliapackaging/package-windows-x86_64:${DOCKER_TAG?}" - always-pull: true - command: ["bash", ".buildkite/utilities/upload_julia.sh"] - propagate-environment: true - volumes: - # Mount buildkite-agent as well - - "C:\\buildkite-agent\\bin:C:\\buildkite-agent\\bin" - environment: - # We have to list these here, because buildkite doesn't automatically - # include environment-hook-set variables in a way that the docker - # plugin finds. It's annoying, but at least we have a workaround. - - "JULIA_CPU_THREADS" - # Have to include this for `buildkite-agent` to work: - - "BUILDKITE_AGENT_ACCESS_TOKEN" - # Have to include these for codesigning and uploading - - "AWS_ACCESS_KEY_ID" - - "AWS_SECRET_ACCESS_KEY" - - "AZURE_TENANT_ID" - - "AZURE_CLIENT_ID" - - "AZURE_CLIENT_SECRET" - # These control where we upload, important when running on julia-buildkite - - "S3_BUCKET" - - "S3_BUCKET_PREFIX" - timeout_in_minutes: ${TIMEOUT?} - agents: - queue: "build" - os: "windows" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/misc/oidc_smoke_test.yml b/pipelines/misc/oidc_smoke_test.yml new file mode 100644 index 000000000..85b7d64d9 --- /dev/null +++ b/pipelines/misc/oidc_smoke_test.yml @@ -0,0 +1,38 @@ +# OIDC/IAM trust smoke test -- validates role assumption and every trust +# boundary from inside a build pipeline, WITHOUT building julia. +# +# Usage (see also "Validation" in ops/README.md): temporarily replace the +# webUI steps of julia-pr (or julia-ci) with this file's steps and start a +# new build (any recent julia commit); restore the real webUI +# (pipelines/main/0_webui.yml) afterwards. +# +# NOTE: while validating a julia-buildkite revision that julia's +# `.buildkite-external-version` does not point to yet, replace the +# `version:` below with the julia-buildkite commit sha under test. +steps: + - label: ":mag: OIDC trust smoke test" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # The sandbox plugin itself runs on Julia (Sandbox.jl) + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + # The bare agents do not carry the AWS CLI; like the historical + # upload jobs, run inside the `aws_uploader` rootfs, which exists + # to provide aws/git/ssh. + rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v8.1/aws_uploader.x86_64.tar.gz" + rootfs_treehash: "c031b94d209e3e86507aaa9998c1fa3544f32c72" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + commands: "bash .buildkite/utilities/tests/oidc_smoke_test.sh" + timeout_in_minutes: 10 + agents: + queue: "build" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" diff --git a/pipelines/publish-test/0_webui.yml b/pipelines/publish-test/0_webui.yml new file mode 100644 index 000000000..c139090fd --- /dev/null +++ b/pipelines/publish-test/0_webui.yml @@ -0,0 +1,29 @@ +# Steps configured in the `julia-publish-test-nosecrets` pipeline's webUI; +# modifying this file has no effect -- paste it into the webUI when it changes. +# +# This is the ISOLATED, NON-PRODUCTION debug pipeline. It runs the real publish +# flow (download -> KMS-sign -> promote) but against THROWAWAY test KMS keys +# (self-signed certs) and the `julialang-test-publish` bucket -- so it can be +# triggered freely to debug the publish flow without touching any production +# key, bucket, or the public `julia-latest-*` pointers. +# +# Unlike `julia-publish`, this pipeline is NOT a trust boundary: it can only +# assume `julia-oidc-publish-test`, which is scoped to the test keys + test +# bucket. So it does NOT need pull-request builds disabled or branch limiting. +# It is triggered manually (never cross-triggered from julia-ci). +# +# Input is seeded with ops/seed_test_staging.sh; trigger a build on the same +# (real master) commit so verify_trusted_commit.sh passes. +steps: + - group: "Publish (test)" + steps: + - label: "Launch publish-test jobs" + agents: + queue: "default" + os: "linux" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/publish-test/launch.yml diff --git a/pipelines/publish-test/launch.yml b/pipelines/publish-test/launch.yml new file mode 100644 index 000000000..76a0b71d0 --- /dev/null +++ b/pipelines/publish-test/launch.yml @@ -0,0 +1,92 @@ +# Launched by the julia-publish-test-nosecrets pipeline. Runs the exact same +# publish.sh flow as production, but the env below redirects every signing key +# to a THROWAWAY *-test KMS key (self-signed cert) and every bucket to the +# isolated julialang-test-publish bucket, and skips the two signers that have +# no non-production equivalent (Windows Azure Trusted Signing, macOS Apple +# notarization). Nothing here can touch a production key, bucket, or pointer. +# +# See ops/terraform/test_publish.tf (test keys + bucket + julia-oidc-publish-test +# role) and ops/seed_test_staging.sh (seed the staged input). +steps: + - group: "Publish (test)" + steps: + - label: ":test_tube: publish all (test)" + key: "publish_all" + concurrency: 1 + concurrency_group: 'publish-test/julialangnightlies' + concurrency_method: eager + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # OUTER julia: runs on the bare agent, only to launch Sandbox.jl + # (the sandbox plugin requires a host julia). NOT visible inside + # the sandbox -- plugins before sandbox#v2 run outside it. + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + # Same toolchain image as the production publish step. + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.6/julia_publish.x86_64.tar.gz + rootfs_treehash: "8b28c5b8937f25827c1ad4b6a3bc1bb90c9e9087" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + # Mount the julia plugin's install + depot so the INNER julia + # plugin below reuses the already-downloaded julia instead of + # re-fetching it inside the sandbox. + - "/cache/julia-buildkite-plugin:/cache/julia-buildkite-plugin" + - JuliaCI/julia#v1: + # INNER julia: plugins after sandbox#v2 run *inside* the sandbox, + # so this puts `julia` on PATH within the rootfs. upload_julia.sh + # uses it to patch pkgimage checksums in the cross-built (foreign + # platform) trees -- the published binaries can't run on this agent. + version: '1.10' # GREP_ME: Keep in sync with the outer julia version. + timeout_in_minutes: 180 + commands: "bash .buildkite/utilities/publish.sh" + agents: + queue: "default" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" + env: + # Scope the test to the Linux + macOS triplets. Linux exercises the + # GPG-sign path (content-agnostic). macOS unpacks the staged unsigned + # Julia.app (assembled by the build_ step), rcodesign-codesigns its + # mach-o with the test KMS key + self-signed cert, runs the cross-mode + # pkgimage checksum patch, and builds the .dmg (notarization skipped). + PUBLISH_ARCHES_FILES: ".buildkite/pipelines/main/platforms/upload_linux.arches .buildkite/pipelines/main/platforms/upload_macos.arches" + # Assume the throwaway test role (test keys + test bucket only). + PUBLISH_OIDC_MODE: "publish-test" + # Read staged input from the julia-ci ephemeral bucket (where a real + # build stages its unsigned tarballs/.app -- the bucket is public-read, + # so the test role can GetObject), but PROMOTE only to the throwaway + # test bucket. So this consumes real staged artifacts and exercises the + # full sign+promote without touching any production release path. + STAGING_BUCKET: "julialang-ephemeral-ci" + S3_BUCKET: "julialang-test-publish" + S3_BUCKET_PREFIX: "bin" + # The test bucket disables object ACLs (BucketOwnerEnforced, public + # read via bucket policy), so the promote must not send --acl. + UPLOAD_TO_S3_ACL: "none" + # Allow re-publishing the same commit repeatedly ("test to our heart's + # content"): overwrite unconditionally instead of write-once. Safe only + # because the throwaway test role permits unconditional PutObject; the + # production publish role's IAM denies it regardless of this flag. + UPLOAD_TO_S3_OVERWRITE: "1" + # Throwaway signing material (test KMS keys + self-signed cert/pubkey). + MACOS_CODESIGN_KMS_KEY: "alias/julia-macos-codesigning-test" + MACOS_CODESIGN_CERT: ".buildkite/utilities/macos/developer_id_test.pem" + TARBALL_SIGNING_KMS_KEY: "alias/julia-tarball-signing-test" + # Derive the GPG pubkey identity from KMS at runtime (GetPublicKey + + # DescribeKey for the creation date) instead of committing a throwaway + # test pubkey. Empty PUBKEY selects derive mode. + TARBALL_SIGNING_PUBKEY: "" + # The two signers with no non-production equivalent are skipped: + # Windows Authenticode (Azure Trusted Signing) and macOS notarization + # (Apple App Store Connect). The .dmg is still built and codesigned. + PUBLISH_SKIP_WINDOWS_SIGN: "1" + PUBLISH_SKIP_NOTARIZATION: "1" + # Inno Setup prefix baked into the rootfs (as in production publish). + WINEPREFIX: "/home/juliaci/.wine" diff --git a/pipelines/publish/0_webui.yml b/pipelines/publish/0_webui.yml new file mode 100644 index 000000000..09bc77328 --- /dev/null +++ b/pipelines/publish/0_webui.yml @@ -0,0 +1,33 @@ +# This file records the steps configured in the julia-publish pipeline's +# webUI; modifying this file has no effect -- paste it into the webUI when +# it changes. +# +# SECURITY: the julia-publish pipeline is the trust boundary. It MUST be +# configured in Buildkite with: +# * Trigger builds after pushing code: OFF +# * Build pull requests: OFF (including from third-party forked +# repositories) +# * Branch limiting: master release-* (and build tags v*) +# Builds are created only by the `trigger` step at the end of julia-ci +# (the inlined wait+trigger at the end of pipelines/main/launch_untrusted_builders.yml). +# Because a pull request can never produce a build under this pipeline slug, +# the slug itself is what the julia-oidc-publish / julia-oidc-docs-deploy IAM +# roles trust -- not the (PR-spoofable) branch name. The publish jobs also +# run verify_trusted_commit.sh as a backstop. +# +# This webUI step loads its launch steps from a pinned julia-buildkite (via +# the external-buildkite plugin), NOT from the triggered build's tree, so a +# malicious build commit cannot alter the publish steps. +steps: + - group: "Publish" + steps: + - label: "Launch publish jobs" + agents: + queue: "default" + os: "linux" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/publish/launch.yml diff --git a/pipelines/publish/launch.yml b/pipelines/publish/launch.yml new file mode 100644 index 000000000..132952544 --- /dev/null +++ b/pipelines/publish/launch.yml @@ -0,0 +1,90 @@ +# Launched by the julia-publish pipeline. A single trusted step signs and +# promotes every staged build to its final release location, then deploys +# docs. This is the only place release secrets (KMS signing keys) are used. +steps: + - group: "Publish" + steps: + - label: ":lock: publish all" + key: "publish_all" + # No in-pipeline build dependency: reads the unsigned tarballs from + # the commit-sha-gated staging area in S3 (written by the build + # pipeline). One job, all triplets. + concurrency: 1 + concurrency_group: 'publish/julialangnightlies' + concurrency_method: eager + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # OUTER julia: runs on the bare agent, only to launch Sandbox.jl + # (the sandbox plugin requires a host julia). Plugins *before* + # sandbox#v2 run outside the sandbox, so this one is NOT visible + # inside it -- the inner julia#v1 below provides the in-sandbox julia. + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + # Run inside the julia_publish rootfs, which carries the full + # signing/packaging toolchain (aws CLI, p7zip, libdmg-hfsplus + + # newfs_hfs, wine + Inno Setup, jsign, python). Without this the + # step runs on a bare agent and fails with `aws: command not + # found`. x86_64-only image; built by JuliaCI/rootfs-images. + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.6/julia_publish.x86_64.tar.gz + rootfs_treehash: "8b28c5b8937f25827c1ad4b6a3bc1bb90c9e9087" + uid: 1000 + gid: 1000 + workspaces: + # Mount the git object cache so upload_julia.sh's version + # introspection (git describe) resolves cleanly inside the + # sandbox, same as the build_linux step. + - "/cache/repos:/cache/repos" + # Mount the julia plugin's install + depot so the inner julia + # plugin below reuses the already-downloaded julia instead of + # re-fetching it inside the sandbox. + - "/cache/julia-buildkite-plugin:/cache/julia-buildkite-plugin" + - JuliaCI/julia#v1: + # INNER julia: plugins after sandbox#v2 run *inside* the sandbox, + # so this puts `julia` on PATH within the rootfs. upload_julia.sh + # uses it to patch the pkgimage checksums inside the (foreign- + # platform) signed install trees -- the published binaries cannot + # run on this agent. + version: '1.10' # GREP_ME: Keep in sync with the outer julia version. + timeout_in_minutes: 180 + commands: "bash .buildkite/utilities/publish.sh" + agents: + # The signing/packaging toolchain is provided by the julia_publish + # rootfs via the sandbox plugin above (rcodesign is fetched at + # runtime). The agent only needs to be able to nest a Sandbox.jl + # sandbox, hence sandbox_capable. + queue: "default" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" + env: + # The Inno Setup 6 prefix is baked into the julia_publish rootfs at + # /home/juliaci/.wine (owned by uid 1000). upload_julia.sh runs + # `wine ISCC.exe` without setting WINEPREFIX, so it would otherwise + # default to $HOME/.wine -- pin it here so the baked prefix is used + # regardless of what HOME the sandbox sets. + WINEPREFIX: "/home/juliaci/.wine" + # Identify the Azure Trusted Signing app registration for Windows + # Authenticode signing (not secrets; auth is via Buildkite OIDC + # workload identity federation -- see windows/codesign.sh and + # ops/terraform/azure). + AZURE_TENANT_ID: "8d3de454-9fdc-4091-90af-9cf79dcabe79" + AZURE_CLIENT_ID: "a1fa180a-5312-4673-a98c-7cde383bae9c" + + # Launcher only: uploads the docs-deploy pipeline and touches no AWS, so + # it must NOT carry key "deploy_docs" -- that key belongs to the actual + # role-assuming "upload docs" job (see misc/deploy_docs.yml), and a + # duplicate key across the build is rejected. + - label: "deploy docs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: "buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml" + agents: + queue: "default" + os: "linux" + arch: "x86_64" diff --git a/pipelines/scheduled/0_webui.yml b/pipelines/scheduled/0_webui.yml index f17513039..4d0087321 100644 --- a/pipelines/scheduled/0_webui.yml +++ b/pipelines/scheduled/0_webui.yml @@ -1,30 +1,23 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +# This file records the steps configured in the Buildkite webUI; modifying +# this file has no effect -- paste it into the webUI when it changes. +# +# There is no secrets management here anymore: jobs that need credentials +# obtain them at runtime via Buildkite OIDC (see utilities/aws_oidc.sh), +# and AWS IAM decides what each job identity may do. The signed/unsigned +# split between pipeline FILES is therefore gone; the trust split lives at +# the level of Buildkite PIPELINES instead (untrusted julia-pr / julia-ci +# vs the trusted julia-publish -- see ops/README.md). steps: - group: "Launch scheduled jobs" steps: - - label: "Unlock secrets, launch pipelines" + - label: "Launch pipelines" agents: - queue: "launch" - cryptic_capable: "true" + queue: "build" os: "linux" plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml - - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/scheduled/launch_signed_jobs.yml - signature_file: .buildkite/pipelines/scheduled/launch_signed_jobs.yml.signature - command: "true" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/launch_untrusted_builders.yml + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/launch_coverage.yml diff --git a/pipelines/scheduled/coverage/coverage.yml b/pipelines/scheduled/coverage/coverage.yml index 132293046..0dbb93d69 100644 --- a/pipelines/scheduled/coverage/coverage.yml +++ b/pipelines/scheduled/coverage/coverage.yml @@ -1,9 +1,16 @@ steps: - group: "Coverage" + # Scheduled (julia-ci) builds, plus "needs full CI"-labeled PRs. Only + # julia-ci holds the codecov / coveralls bearer tokens; PR builds run + # the same coverage collection but skip the uploads (the lcov tarball + # is still available as a buildkite artifact). + if: (build.source == "schedule") || (build.pull_request.labels includes "needs full CI") + notify: + - github_commit_status: + context: "Coverage" steps: - label: ":linux: coverage (x86_64)" key: "coverage-linux" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -12,17 +19,27 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 - - JuliaCI/cryptic#v2: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - timeout_in_minutes: 360 # 6h + timeout_in_minutes: 600 # 10h commands: | + echo "--- Obtain coverage tokens" + # Coverage tokens are fetched from SSM Parameter Store with this + # job's OIDC identity; no static secrets exist in CI config. + # Only julia-ci holds tokens (PR builds and the julia-buildkite + # self-test have no tokens role): collect coverage anyway, skip + # the uploads below. + if [[ "$${BUILDKITE_PIPELINE_SLUG}" != "julia-ci" ]]; then + echo "No coverage tokens in this pipeline; coverage uploads will be skipped" + else + source .buildkite/utilities/aws_oidc.sh tokens + export CODECOV_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/codecov_token --query Parameter.Value --output text)" + export COVERALLS_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text)" + fi + echo "--- Build Julia from source" make --output-sync -j$${JULIA_NUM_THREADS} @@ -49,7 +66,11 @@ steps: buildkite-agent artifact upload lcov_files.tar.gz echo "--- Process and upload coverage information" - ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + if [[ -n "$${CODECOV_TOKEN:-}" ]]; then + ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + else + echo "No coverage tokens; skipping upload" + fi agents: queue: "test" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -57,14 +78,11 @@ steps: os: "linux" arch: "x86_64" env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} - label: ":macos: coverage (aarch64)" key: "coverage-macos-aarch64" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -73,12 +91,22 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - JuliaCI/cryptic#v2: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" timeout_in_minutes: 300 # 5h commands: | + echo "--- Obtain coverage tokens" + # Coverage tokens are fetched from SSM Parameter Store with this + # job's OIDC identity; no static secrets exist in CI config. + # Only julia-ci holds tokens (PR builds and the julia-buildkite + # self-test have no tokens role): collect coverage anyway, skip + # the uploads below. + if [[ "$${BUILDKITE_PIPELINE_SLUG}" != "julia-ci" ]]; then + echo "No coverage tokens in this pipeline; coverage uploads will be skipped" + else + source .buildkite/utilities/aws_oidc.sh tokens + export CODECOV_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/codecov_token --query Parameter.Value --output text)" + export COVERALLS_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text)" + fi + echo "--- Build Julia from source" make -j$${JULIA_NUM_THREADS} @@ -105,21 +133,22 @@ steps: buildkite-agent artifact upload lcov_files.tar.gz echo "--- Process and upload coverage information" - ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + if [[ -n "$${CODECOV_TOKEN:-}" ]]; then + ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + else + echo "No coverage tokens; skipping upload" + fi agents: queue: "test" os: "macos" arch: "aarch64" env: JULIA_SHELL: "/bin/bash" - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} - label: ":windows: coverage (x86_64)" key: "coverage-windows-x86_64" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -128,14 +157,18 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - JuliaCI/cryptic#v2: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - docker#v3.13.0: - image: "juliapackaging/package-windows-x86_64:v7.10" + image: "juliapackaging/package-windows-x86_64:v8.5" always-pull: true command: ["bash", "-c", " + echo '--- Obtain coverage tokens' && + if [[ $${BUILDKITE_PIPELINE_SLUG} == julia-pr ]]; then + echo 'PR build: no coverage tokens; coverage uploads will be skipped'; + else + source .buildkite/utilities/aws_oidc.sh tokens && + export CODECOV_TOKEN=$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/codecov_token --query Parameter.Value --output text) && + export COVERALLS_TOKEN=$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text); + fi && echo '--- Build Julia from source' && make -j$${JULIA_NUM_THREADS} VERBOSE=1 && echo '--- Print Julia version info' && @@ -156,7 +189,11 @@ steps: tar -zcf lcov_files.tar.gz lcov_files/ && buildkite-agent artifact upload lcov_files.tar.gz && echo '--- Process and upload coverage information' && - ./usr/bin/julia.exe --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + if [[ -n $${CODECOV_TOKEN:-} ]]; then + ./usr/bin/julia.exe --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl; + else + echo 'No coverage tokens (PR build); skipping upload'; + fi "] propagate-environment: true volumes: @@ -174,9 +211,8 @@ steps: - "BUILDKITE_BUILD_AUTHOR_EMAIL" - "BUILDKITE_BUILD_CREATOR_EMAIL" - "BUILDKITE_PULL_REQUEST" - - "CODECOV_TOKEN" - - "COVERALLS_TOKEN" - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_PIPELINE_SLUG" timeout_in_minutes: 540 # 9h agents: queue: "test" @@ -184,14 +220,11 @@ steps: arch: "x86_64" env: JULIA_SHELL: "/bin/bash" - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} - label: "Finalize parallel coverage" key: "coverage-finalize" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") depends_on: - step: "coverage-linux" allow_failure: true @@ -207,19 +240,24 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - JuliaCI/cryptic#v2: - variables: - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" timeout_in_minutes: 10 commands: | echo "--- Signal completion of parallel coverage uploads" - julia --color=yes .buildkite/pipelines/scheduled/coverage/finish_parallel_coverage.jl + # PR builds hold no tokens and did not upload anything, so there + # is nothing to finalize there. + if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-pr" ]]; then + echo "PR build: nothing was uploaded; nothing to finalize" + else + # The coveralls token is fetched from SSM Parameter Store with + # this job's OIDC identity; no static secrets exist in CI config. + source .buildkite/utilities/aws_oidc.sh tokens + export COVERALLS_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text)" + julia --color=yes .buildkite/pipelines/scheduled/coverage/finish_parallel_coverage.jl + fi agents: queue: "test" os: "linux" arch: "x86_64" env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature deleted file mode 100644 index 71698ff81..000000000 --- a/pipelines/scheduled/coverage/coverage.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__aMmratO(\SjAgB@QYJ7Ab)qRkjq{w]$"B֧̈́ \ No newline at end of file diff --git a/pipelines/scheduled/coverage/upload_coverage.jl b/pipelines/scheduled/coverage/upload_coverage.jl index 08b90360b..138047188 100644 --- a/pipelines/scheduled/coverage/upload_coverage.jl +++ b/pipelines/scheduled/coverage/upload_coverage.jl @@ -4,6 +4,7 @@ import Pkg import Logging import TOML +Pkg.develop(path="./JuliaSyntax") Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1") import Coverage diff --git a/pipelines/scheduled/launch_coverage.yml b/pipelines/scheduled/launch_coverage.yml new file mode 100644 index 000000000..eb2786f89 --- /dev/null +++ b/pipelines/scheduled/launch_coverage.yml @@ -0,0 +1,28 @@ +# Launch the scheduled coverage pipeline. +# +# Historically this launched "signed pipelines" (treehash-verified by the +# cryptic plugin) because they carried decryption rights for static +# secrets. There are no pipeline-held secrets anymore: jobs obtain +# credentials at runtime via Buildkite OIDC, and AWS IAM decides what each +# job identity (pipeline/ref/step) may do. +# +# The no-GPL upload relay jobs are gone too: the no-GPL build steps stage +# their unsigned tarballs directly to the julia-ci ephemeral staging bucket +# (under the bin-nogpl prefix; see build_envs.sh). Signing + promotion to +# the julialang-nogpl bucket belongs in the julia-publish pipeline (TODO: +# wire the no-GPL triplets into utilities/publish.sh). +steps: + - group: "Launch scheduled jobs" + steps: + - label: "Launch upload jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Launch the code coverage job + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/coverage/coverage.yml + agents: + queue: "build" + os: "linux" + arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/scheduled/launch_signed_jobs.yml b/pipelines/scheduled/launch_signed_jobs.yml deleted file mode 100644 index 88e87863a..000000000 --- a/pipelines/scheduled/launch_signed_jobs.yml +++ /dev/null @@ -1,47 +0,0 @@ -steps: - - group: "Launch scheduled jobs" - steps: - - label: "Launch signed jobs" - command: "true" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - verbose: true - signed_pipelines: - # Launch the code coverage job - - pipeline: .buildkite/pipelines/scheduled/coverage/coverage.yml - signature_file: .buildkite/pipelines/scheduled/coverage/coverage.yml.signature - inputs: - - .buildkite/pipelines/scheduled/coverage/coverage.yml - - .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl - - .buildkite/pipelines/scheduled/coverage/upload_coverage.jl - - .buildkite/pipelines/scheduled/coverage/finish_coveralls_parallel.jl - # Launch the upload jobs - - pipeline: .buildkite/pipelines/scheduled/launch_upload_jobs.yml - signature_file: .buildkite/pipelines/scheduled/launch_upload_jobs.yml.signature - inputs: - - .buildkite/pipelines/main/launch_upload_jobs.yml - - .buildkite/utilities/arches_pipeline_upload.sh - - .buildkite/utilities/arches_env.sh - - .buildkite/utilities/build_envs.sh - - .buildkite/utilities/extract_triplet.sh - - .buildkite/utilities/upload_julia.sh - - .buildkite/utilities/sign_tarball.sh - - .buildkite/utilities/macos/build_dmg.sh - - .buildkite/utilities/macos/codesign.sh - - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml - - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches - - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml - - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches - - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml - - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/scheduled/launch_signed_jobs.yml.signature b/pipelines/scheduled/launch_signed_jobs.yml.signature deleted file mode 100644 index f6eacf665..000000000 --- a/pipelines/scheduled/launch_signed_jobs.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__UXzz!Fh@DAP&)$%2ޱß0)XkN},F04ER^jLL}C|]I \ No newline at end of file diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_untrusted_builders.yml similarity index 91% rename from pipelines/scheduled/launch_unsigned_jobs.yml rename to pipelines/scheduled/launch_untrusted_builders.yml index b7bd5566f..e8084bab6 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_untrusted_builders.yml @@ -1,10 +1,10 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. +# This file launches all the UNTRUSTED build + test jobs; no job in the +# build pipelines holds secrets or signing rights (the trusted work happens +# in the separate julia-publish Buildkite pipeline). # # Yes, this is creating another layer of indirection; the flow now looks like: # -# [webui] -> launch_unsigned_builders.yml -> misc/llvmpasses.yml +# [webui] -> launch_untrusted_builders.yml -> misc/llvmpasses.yml # # when we could theoretically just have the `webui` launch `misc/llvmpasses.yml`, # however this raises the bar for contributors to add new (unsigned) steps to diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml deleted file mode 100644 index f8016246d..000000000 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 - -steps: - - group: "Upload (no GPL)" - steps: - - label: "Launch upload jobs (no GPL)" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - - # Launch `upload_*` jobs to store tarballs into S3 once tests are done - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml - - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/scheduled/launch_upload_jobs.yml.signature b/pipelines/scheduled/launch_upload_jobs.yml.signature deleted file mode 100644 index 20a19ebac..000000000 Binary files a/pipelines/scheduled/launch_upload_jobs.yml.signature and /dev/null differ diff --git a/pipelines/scheduled/platforms/upload_linux.no_gpl.yml b/pipelines/scheduled/platforms/upload_linux.no_gpl.yml deleted file mode 100644 index 099aeee96..000000000 --- a/pipelines/scheduled/platforms/upload_linux.no_gpl.yml +++ /dev/null @@ -1,56 +0,0 @@ -steps: - - group: "Upload (no GPL)" - steps: - - label: ":linux: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - # We only upload to S3 if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite") || (pipeline.slug == "julia-buildkite-scheduled") || (build.pull_request.labels includes "needs full CI") - depends_on: - # Wait for the build to finish - - "build_${TRIPLET?}" - # There are no tests to wait on - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/aws_uploader.x86_64.tar.gz - rootfs_treehash: "61cb30497604eb1581b3ffd6a8058c150f4cbf8b" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - files: - - .buildkite/secrets/tarball_signing.gpg - timeout_in_minutes: ${TIMEOUT?} - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox_capable: "true" - os: "linux" - arch: "x86_64" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - S3_BUCKET: "julialang-nogpl" - S3_BUCKET_PREFIX: "bin-nogpl" diff --git a/pipelines/scheduled/platforms/upload_macos.no_gpl.yml b/pipelines/scheduled/platforms/upload_macos.no_gpl.yml deleted file mode 100644 index e3c40cd18..000000000 --- a/pipelines/scheduled/platforms/upload_macos.no_gpl.yml +++ /dev/null @@ -1,45 +0,0 @@ -steps: - - group: "Upload (no GPL)" - steps: - - label: ":macos: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - # We only upload to S3 if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite") || (pipeline.slug == "julia-buildkite-scheduled") || (build.pull_request.labels includes "needs full CI") - depends_on: - # Wait for the build to finish - - "build_${TRIPLET?}" - # There are no tests to wait on - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - NOTARIZATION_APPLE_ID="U2FsdGVkX1/b2eyuYolwIaW0DI2YE5tPcMYJ7X7/J5Qz6T/oV6bZEuQqUEet6Jvx" - - NOTARIZATION_APPLE_KEY="U2FsdGVkX1+z0XUSEnj0vLawcT1/U2wx1YDN6s3V9U5+qhbGwRQuXHNJXPmlNe/o" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/macos_codesigning.keychain - timeout_in_minutes: ${TIMEOUT?} - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - os: "macos" - arch: "${ARCH}" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - S3_BUCKET: "julialang-nogpl" - S3_BUCKET_PREFIX: "bin-nogpl" diff --git a/pipelines/scheduled/platforms/upload_windows.no_gpl.yml b/pipelines/scheduled/platforms/upload_windows.no_gpl.yml deleted file mode 100644 index c93485cfe..000000000 --- a/pipelines/scheduled/platforms/upload_windows.no_gpl.yml +++ /dev/null @@ -1,69 +0,0 @@ -steps: - - group: "Upload (no GPL)" - steps: - - label: ":windows: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - # We only upload to S3 if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite-scheduled") || (build.pull_request.labels includes "needs full CI") - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw==" - - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ==" - - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA==" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/windows_codesigning.pfx - - docker#v3.13.0: - # We always use `x86_64` here, as it doesn't matter what the compiler is for uploading - image: "juliapackaging/package-windows-x86_64:${DOCKER_TAG?}" - always-pull: true - command: ["bash", ".buildkite/utilities/upload_julia.sh"] - propagate-environment: true - volumes: - # Mount buildkite-agent as well - - "C:\\buildkite-agent\\bin:C:\\buildkite-agent\\bin" - environment: - # We have to list these here, because buildkite doesn't automatically - # include environment-hook-set variables in a way that the docker - # plugin finds. It's annoying, but at least we have a workaround. - - "JULIA_CPU_THREADS" - # Have to include this for `buildkite-agent` to work: - - "BUILDKITE_AGENT_ACCESS_TOKEN" - # Have to include these for codesigning and uploading - - "AWS_ACCESS_KEY_ID" - - "AWS_SECRET_ACCESS_KEY" - - "AZURE_TENANT_ID" - - "AZURE_CLIENT_ID" - - "AZURE_CLIENT_SECRET" - # These control where we upload - must be explicit values for no_gpl - # (pass-through would use the hook-set ephemeral bucket instead) - - "S3_BUCKET=julialang-nogpl" - - "S3_BUCKET_PREFIX=bin-nogpl" - timeout_in_minutes: ${TIMEOUT?} - agents: - queue: "build" - os: "windows" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - S3_BUCKET: "julialang-nogpl" - S3_BUCKET_PREFIX: "bin-nogpl" diff --git a/port_oidc_1.13.sh b/port_oidc_1.13.sh new file mode 100644 index 000000000..2395bbf56 --- /dev/null +++ b/port_oidc_1.13.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Port the #544 OIDC/KMS trust architecture from `main` onto release-julia-1.13, +# reusing the shared julia-pr / julia-ci / julia-publish pipeline slugs. +# +# Principle: make the tree match `main` (the target architecture, already +# designed to serve `master` AND `release-*`) EXCEPT the genuinely +# release-specific build matrix (the *.arches files + launch_powerpc.jl), +# which are preserved from release-julia-1.13. +set -euo pipefail +cd "$(dirname "$0")" + +BASE=origin/release-julia-1.13 +MAIN=origin/main +BR=kf/port-oidc-release-1.13 + +git rev-parse --verify "$BR" >/dev/null 2>&1 && git branch -D "$BR" +git checkout -q -b "$BR" "$BASE" + +# 1. Overlay main's tree wholesale (every path that exists in main): shared +# files get main's version, main-only files are added. Paths that exist +# only on release are left untouched here (removed in step 3). +git checkout -q "$MAIN" -- . + +# 2. Restore the release-1.13 build matrix (rootfs v6.00/v7.10, mmtk on +# package_linux, Windows CFLAGS) + the powerpc launcher. +mapfile -t KEEP < <(git ls-tree -r --name-only "$BASE" -- \ + pipelines/main/platforms pipelines/scheduled/platforms \ + | grep -E '\.arches$') +KEEP+=(pipelines/main/platforms/launch_powerpc.jl) +git checkout -q "$BASE" -- "${KEEP[@]}" + +# 3. Retire the pre-#544 cryptic / signed-split / secrets infrastructure. +git rm -q -r --ignore-unmatch \ + cryptic_repo_keys secrets .buildkite/cryptic_repo_root \ + devdocs/sign.md \ + utilities/aws_config utilities/deploy_docs.sh utilities/sign_tarball.sh \ + pipelines/main/launch_signed_jobs.yml pipelines/main/launch_signed_jobs.yml.signature \ + pipelines/main/launch_unsigned_builders.yml pipelines/main/launch_unsigned_jobs.yml \ + pipelines/main/launch_upload_jobs.yml pipelines/main/launch_upload_jobs.yml.signature \ + pipelines/main/misc/upload_buildkite_results.yml \ + pipelines/main/platforms/upload_freebsd.yml pipelines/main/platforms/upload_linux.yml \ + pipelines/main/platforms/upload_macos.yml pipelines/main/platforms/upload_windows.yml \ + pipelines/scheduled/coverage/coverage.yml.signature \ + pipelines/scheduled/launch_signed_jobs.yml pipelines/scheduled/launch_signed_jobs.yml.signature \ + pipelines/scheduled/launch_unsigned_jobs.yml pipelines/scheduled/launch_upload_jobs.yml \ + pipelines/scheduled/launch_upload_jobs.yml.signature \ + pipelines/scheduled/platforms/upload_linux.no_gpl.yml \ + pipelines/scheduled/platforms/upload_macos.no_gpl.yml \ + pipelines/scheduled/platforms/upload_windows.no_gpl.yml \ + >/dev/null + +echo "Port applied on branch $BR (renderer trim happens separately)." diff --git a/secrets/.gitignore b/secrets/.gitignore deleted file mode 100644 index 2a84f4868..000000000 --- a/secrets/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Ignore everything -* - -# Don't ignore this `.gitignore` file -!.gitignore - -# Don't ignore encrypted files -!*.encrypted - -# Don't ignore public keys, that's fine to include -!*.pub diff --git a/secrets/macos_codesigning.keychain.encrypted b/secrets/macos_codesigning.keychain.encrypted deleted file mode 100644 index b27756768..000000000 Binary files a/secrets/macos_codesigning.keychain.encrypted and /dev/null differ diff --git a/secrets/ssh_docs_deploy.encrypted b/secrets/ssh_docs_deploy.encrypted deleted file mode 100644 index 8b7e2ffe2..000000000 Binary files a/secrets/ssh_docs_deploy.encrypted and /dev/null differ diff --git a/secrets/ssh_docs_deploy.pub b/secrets/ssh_docs_deploy.pub deleted file mode 100644 index eaea073e5..000000000 --- a/secrets/ssh_docs_deploy.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2HupO7+uq6NE//ZCCYS1szwDVutH8ZVtF0wjJJGmfRLzDe6l2Kcx+CY+i3k2HoxrfNlmri3RdWrfDbEruPZlWaz9HH/Hi8S8ZkyQO932dbag7u5JGKw3Mb/3x05O2QaL+0HGItUyfFMFT9NBZ1na+AH/ZPWamXR98PLh39ic1HFw2x2hacYA/4w0ylxwrojRfCqcjK/YVJUCdQ/XwsmSjs+0+rIfdVwSLbJKeHj5JYLX6CmF4zf4WzJKKDXx1k6gwaSS6oY5XOVit2I1u80cxZRiQhrMfYPKywY5+Y6gqjrGABLYSq/JJRKsgdJxs39V8O1ZjXVsGxbR+1r3F9ISH buildkite-docs-deploy diff --git a/secrets/tarball_signing.gpg.encrypted b/secrets/tarball_signing.gpg.encrypted deleted file mode 100644 index 323e8fc1a..000000000 Binary files a/secrets/tarball_signing.gpg.encrypted and /dev/null differ diff --git a/secrets/windows_codesigning.pfx.encrypted b/secrets/windows_codesigning.pfx.encrypted deleted file mode 100644 index f8ee914db..000000000 Binary files a/secrets/windows_codesigning.pfx.encrypted and /dev/null differ diff --git a/signing-pubkeys/.gitignore b/signing-pubkeys/.gitignore new file mode 100644 index 000000000..6741e9e01 --- /dev/null +++ b/signing-pubkeys/.gitignore @@ -0,0 +1,12 @@ +# Ignore everything +* + +# Don't ignore this `.gitignore` file +!.gitignore + +# Don't ignore public keys, that's fine to include +!*.pub + +# The exported OpenPGP public half of the KMS release signing key +# (see ops/20_export_gpg_pubkey.py); contains no secret material. +!tarball_signing.pub.asc diff --git a/signing-pubkeys/tarball_signing.pub.asc b/signing-pubkeys/tarball_signing.pub.asc new file mode 100644 index 000000000..57650752e --- /dev/null +++ b/signing-pubkeys/tarball_signing.pub.asc @@ -0,0 +1,28 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +xsFNBGoseh4BEADemCYuKp7uLR304Eij5hYPXHhl0FuOsOvWBJJal89SYRp0r7ix +uxkotCtMeQLBjj3n/sDFRYORT7Hil0Aylp58J4ZInHJutmpEm2V/AijyLcrmeGXJ +QiCF3rJzFyTgr7MAXwzpq7KtmJlXZQzGUknsnNs9PzLFm81qvchpsmW1ZE8pOzz+ +wOItlN7HnbZpdmQwRPRkp5X+FePqT4ICci/ZN88SkqE5gHStxPvUbm8Vlg4CJ6bp +TG7HnXda6X1CWCa/5+Z0PwsThHipZ1Yv/Snxm5V3BmI5LiqQbESyGezRs96wbBOI +o2VTqp6yvGCt85XNGGsmuH+L69lEzdN8SxQwPN8rS2OBocWilGabl/RtfIzKLH16 +PmrfAuuNzA+TGFpztoFMZVt3jCL0CWNJALM9vd/vevuhuTlyXZ9JjbYplEQFy0e+ +9IHvXCsYB1MKPmL8ZlbVqRJ0niTsfCDs8+H2dO2md/x5JJPBY58nk+wFnTEmV/sN +WX5f6CrZUfFeGmmXoRpdbodtx0p7zrYi9kC6hUYS+wffsNeAkz3wWQ9bOPRrWUga +xwPln8CVQcnlQF37NiEHgtg/xF6HOE15IkVR6x+JfrrmjHslhX5YLvRLAYCFUPmb +r87tVdFkCafwitTh58rvY3ZeF/WHdUuqmrtweFRRjICa3qQAvACh94QiuwARAQAB +zTJKdWxpYSBSZWxlYXNlIFNpZ25pbmcgS2V5IDxidWlsZGJvdEBqdWxpYWxhbmcu +b3JnPsLBdgQTAQgAIBYhBGS3eaVwly//e/wrVOrUceGh8sEKBQJqLHoeAhsDAAoJ +EOrUceGh8sEKe1YP/23Qp9jFyLhyK+OKkGvt+REgKSB4UKOTxpsc4HvUFvUUB4Mb +9AjBdzUPGavOSMbkDytZNz7yFrb/rWfwd5tgMP973xBYzipztIj3MsQC8X7Qg8sS +rxdh37+2G4di34/AIo0HuPIGwne9hJXxyPa+Opj8DR7K8dC+aeMOydewUVoNQhz4 +Wnuir+LryM4Ubs0+pWYTTU51tj8ADgirWDz84ZajLwANAPkYNdgsfnk7eV05/gc3 +GC8ygahSZgTfnwojwvKoFfLDsM6z0zjTryoJY68xe3NYh59kdEN0W8jKw3OJGMcX +T82zckNSwAEIJOdMaqauRAHeGJ/kPovyy3enxsq8VKA9eBAxlmlUcuodnLCZHI+d +g/VKMUPegMuvyaZnY2VAHBmONKX4jpeMZYbRYfyXa2InixN/hcaaA3g/90yevCH+ +GNmN1MaXfkjISUdsMvLQMixVeQ0/tm+SHSopFwTJtuzEaAyyNAkLL4U1iRRSMwW1 +BaJrc943PhBQ4ytkWJhm0ge1e2qRB3SSKYzoOTji98hA/hUPmt8ZO+yb8FngJXbw ++fa+onWVxbGp8AO8+TNvJb89iuCC/VoMtf9sdSnZ85Zj3+jJk+aVauQHohgfwVOX +eBjaNN1vJCy+1efZArpk6d6H94ZeCMLnEzxzVQz+3AF1BgZVQNF8hPD2qiK0 +=uJpw +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utilities/arches_env.sh b/utilities/arches_env.sh index ff02e5dce..69f4e0eaf 100755 --- a/utilities/arches_env.sh +++ b/utilities/arches_env.sh @@ -15,7 +15,7 @@ fi enforce_sanitized() { for value in "$@"; do - if ! [[ "${value}" =~ ^[a-zA-Z0-9_\.[:space:]=,-]*$ ]]; then + if ! [[ "${value}" =~ ^[a-zA-Z0-9_\.\"+[:space:]=,-]*$ ]]; then echo "Arches file '${ARCHES_FILE}' contains value '${value}' with non-alphanumeric characters; refusing to parse!" >&2 exit 1 fi @@ -54,7 +54,13 @@ while read -r line; do enforce_sanitized "${line}" # Convert line to array - readarray -d ' ' -t line_array <<<"${line}" + { + set -f + # shellcheck disable=SC2207 + # `read -a` ignores quoted strings when splitting and we are controlling word + # splitting via IFS and `set -f` as ShellCheck recommends + IFS=$'\n' line_array=( $(xargs -n1 <<<"${line}") ) + } # Panic if we don't have the same number of items as our column names: if [[ "${#line_array[@]}" != "${#var_names[@]}" ]]; then diff --git a/utilities/aws_config b/utilities/aws_config deleted file mode 100644 index fd9c55742..000000000 --- a/utilities/aws_config +++ /dev/null @@ -1,4 +0,0 @@ -[profile s3copy] -credential_source=Environment -s3= - multipart_threshold=5GB diff --git a/utilities/aws_oidc.sh b/utilities/aws_oidc.sh new file mode 100755 index 000000000..f640f8a4a --- /dev/null +++ b/utilities/aws_oidc.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Obtain AWS credentials for this Buildkite job via OIDC web identity +# federation. No static AWS secrets are involved: the job's identity +# (pipeline, ref, commit, step) is attested by Buildkite and verified by +# AWS IAM, which scopes what the job may do (see ops/ in this repo). +# +# Source this script with the desired role: +# +# source .buildkite/utilities/aws_oidc.sh stage # untrusted: write-once to own pipeline's staging bucket, / path +# source .buildkite/utilities/aws_oidc.sh publish # trusted: sign + promote to final (publish pipeline only) +# source .buildkite/utilities/aws_oidc.sh docs-deploy # trusted: docs deploy SSH signing via KMS +# source .buildkite/utilities/aws_oidc.sh tokens # CI telemetry tokens from SSM (julia-ci only) +# +# The untrusted roles exist once per build pipeline (julia-oidc-stage-pr / +# julia-oidc-stage-ci, ...): `stage` resolves to this pipeline's role. +# `tokens` resolves to julia-oidc-tokens-ci and is refused on pull request +# builds -- a PR runs attacker-controlled code inside the job, so PR +# builds get no bearer tokens at all. +# +# The `publish` and `docs-deploy` roles are only assumable from the +# `julia-publish` pipeline (PR builds disabled there); callers should also +# run verify_trusted_commit.sh first as defense in depth. +# +# It exports AWS_WEB_IDENTITY_TOKEN_FILE / AWS_ROLE_ARN, which every AWS +# SDK and the AWS CLI use to assume the role automatically (and refresh +# as needed). The token carries only the AWS session tags that the selected +# role's trust and permission policies match against. +# (Sourced file: deliberately no `set -euo pipefail` here -- shell options +# set in a sourced file leak into the calling script; strict mode belongs +# to the entrypoints.) + +# AWS account that holds the Julia CI roles (not a secret). +JULIA_CI_AWS_ACCOUNT_ID="${JULIA_CI_AWS_ACCOUNT_ID:-873569884612}" +JULIA_CI_AWS_REGION="${JULIA_CI_AWS_REGION:-us-east-1}" + +_OIDC_ROLE_SUFFIX="${1:?usage: source aws_oidc.sh }" + +# The trusted roles must only ever be requested from the dedicated publish +# pipeline. The IAM trust policy already enforces this (it only trusts the +# julia-publish* slug), but refuse early here too so a misconfiguration +# surfaces loudly rather than as a confusing AccessDenied. Pull request +# builds never run in a publish pipeline. +# publish-test resolves to the throwaway, non-production test role +# (julia-oidc-publish-test): it can only sign with the *-test KMS keys and +# read/write the test bucket. Like the production trusted roles it must come +# from a *publish* pipeline slug, but -- being harmless -- it is NOT refused on +# pull-request builds, so the test publish flow can be exercised from anywhere. +case "${_OIDC_ROLE_SUFFIX}" in + publish|docs-deploy|publish-test) + if [[ "${BUILDKITE_PIPELINE_SLUG:-}" != *publish* ]]; then + echo "ERROR: ${_OIDC_ROLE_SUFFIX} role requested from non-publish pipeline '${BUILDKITE_PIPELINE_SLUG:-}'" >&2 + return 1 2>/dev/null || exit 1 + fi + ;;& + publish|docs-deploy) + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + echo "ERROR: ${_OIDC_ROLE_SUFFIX} role must not be requested on a pull request build" >&2 + return 1 2>/dev/null || exit 1 + fi + ;; +esac + +# The untrusted roles exist once per build pipeline; resolve to ours. +case "${_OIDC_ROLE_SUFFIX}" in + stage) + if [[ "${BUILDKITE_PIPELINE_SLUG:-}" == "julia-pr" ]]; then + _OIDC_ROLE_SUFFIX="stage-pr" + elif [[ "${BUILDKITE_PIPELINE_SLUG:-}" == julia-buildkite* ]]; then + # The julia-buildkite repository's own self-test CI. + _OIDC_ROLE_SUFFIX="stage-buildkite" + else + _OIDC_ROLE_SUFFIX="stage-ci" + fi + ;; + tokens) + # Only julia-ci has a tokens role (see ops/terraform/iam.tf): PR + # builds and the julia-buildkite self-test run untrusted code that + # could exfiltrate any bearer token available to the job. + if [[ "${BUILDKITE_PIPELINE_SLUG:-}" != "julia-ci" || "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + echo "ERROR: bearer tokens are only available to julia-ci branch builds" >&2 + return 1 2>/dev/null || exit 1 + fi + _OIDC_ROLE_SUFFIX="tokens-ci" + ;; +esac + +# The *_id tags carry the Buildkite UUIDs (not the renameable slugs); the +# IAM trust policies pin organization_id / pipeline_id / cluster_id so a +# recreated or renamed pipeline with a matching slug cannot assume a role. +case "${_OIDC_ROLE_SUFFIX}" in + stage-pr|stage-ci|stage-buildkite) + # Trust: org/pipeline/cluster IDs. Permission policy: own commit path. + _OIDC_AWS_SESSION_TAGS="organization_id,pipeline_id,cluster_id,build_commit" + ;; + publish|docs-deploy) + # Trust: org/pipeline/cluster IDs and step key. KMS policies also gate + # the trusted roles by step_key. + _OIDC_AWS_SESSION_TAGS="organization_id,pipeline_id,cluster_id,step_key" + ;; + tokens-ci|publish-test) + # Trust only needs the immutable Buildkite IDs. + _OIDC_AWS_SESSION_TAGS="organization_id,pipeline_id,cluster_id" + ;; + *) + echo "ERROR: unknown OIDC role suffix '${_OIDC_ROLE_SUFFIX}'" >&2 + return 1 2>/dev/null || exit 1 + ;; +esac + +# Buildkite caps the OIDC token lifetime at 7200s (2h). The AWS SDK/CLI +# re-assumes the role from the token file whenever the (1h) STS session +# expires, so jobs whose AWS usage spans more than ~2h must re-source +# this script to mint a fresh token (publish.sh does so per triplet). +_OIDC_TOKEN_FILE="$(mktemp)" +# Agent v3.104+ redacts the token from logs via the Job API (a unix socket), +# which is unreachable inside the publish sandbox and aborts the request. We +# write the token to a file and never echo it, so skip redaction -- but only +# on agents new enough to know the flag (older build-cluster agents reject it +# and aren't sandbox-broken). Detect by version: `--help` can't be probed, +# since on affected agents it too needs the Job API and aborts. +_OIDC_SKIP_REDACTION=() +_BK_VER="$(buildkite-agent --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1)" +if [ -n "${_BK_VER}" ] && { [ "${_BK_VER%%.*}" -gt 3 ] || { [ "${_BK_VER%%.*}" -eq 3 ] && [ "${_BK_VER#*.}" -ge 104 ]; }; }; then + _OIDC_SKIP_REDACTION=( --skip-redaction ) +fi +buildkite-agent oidc request-token \ + --audience "sts.amazonaws.com" \ + --lifetime 7200 \ + "${_OIDC_SKIP_REDACTION[@]}" \ + --aws-session-tag "${_OIDC_AWS_SESSION_TAGS}" \ + > "${_OIDC_TOKEN_FILE}" + +export AWS_WEB_IDENTITY_TOKEN_FILE="${_OIDC_TOKEN_FILE}" +export AWS_ROLE_ARN="arn:aws:iam::${JULIA_CI_AWS_ACCOUNT_ID}:role/julia-oidc-${_OIDC_ROLE_SUFFIX}" +export AWS_ROLE_SESSION_NAME="bk-$(tr -dc 'a-zA-Z0-9=,.@-' <<<"${BUILDKITE_STEP_KEY:-job}" | cut -c1-48)-${BUILDKITE_BUILD_NUMBER:-0}" +export AWS_DEFAULT_REGION="${JULIA_CI_AWS_REGION}" +export AWS_REGION="${JULIA_CI_AWS_REGION}" + +# Make sure stale static credentials can never shadow the role. +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + +echo "AWS credentials: ${AWS_ROLE_ARN} (via Buildkite OIDC)" diff --git a/utilities/build_envs.sh b/utilities/build_envs.sh index 1744dacd1..b70aaf38c 100755 --- a/utilities/build_envs.sh +++ b/utilities/build_envs.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# (Sourced file: deliberately no `set` of shell options here -- they would +# leak into the calling script; strict mode belongs to the entrypoints.) # First, extract information from our triplet # shellcheck source=SCRIPTDIR/extract_triplet.sh @@ -77,8 +79,9 @@ case "${ARCH?}" in "carmel,clone_all" # Apple M1 "apple-m1,base(3)" - # Vector-length-agnostic common denominator between Neoverse V1 and V2, recent Arm server architectures - "neoverse-512tvb,base(3)" + # Vector-length-agnostic common denominator between Neoverse V1 and V2, recent Arm server architectures. + # Disable little used rand and fpac features which would prevent matching on many systems. + "neoverse-512tvb,-rand,-fpac,base(3)" ) ;; esac @@ -105,8 +108,6 @@ fi JULIA_CPU_TARGET="$(printf ";%s" "${JULIA_CPU_TARGETS[@]}")" export JULIA_CPU_TARGET="${JULIA_CPU_TARGET:1}" -export JULIA_IMAGE_THREADS="$JULIA_CPU_THREADS" - # Extract git information SHORT_COMMIT_LENGTH=10 @@ -166,14 +167,20 @@ fi export JULIA_INSTALL_DIR="julia-${TAR_VERSION}" JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia${EXE}" -# By default, we upload to `julialangnightlies/bin`, but we allow this to be overridden -S3_BUCKET="${S3_BUCKET:-julialangnightlies}" -S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}" +# By default, we upload to `julialangnightlies/bin` (no-GPL builds to +# `julialang-nogpl/bin-nogpl`), but we allow this to be overridden +if [[ "${OS?}" == *nogpl ]]; then + S3_BUCKET="${S3_BUCKET:-julialang-nogpl}" + S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin-nogpl}" +else + S3_BUCKET="${S3_BUCKET:-julialangnightlies}" + S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}" +fi # We generally upload to multiple upload targets UPLOAD_TARGETS=() -if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release-* ]] || [[ "${BUILDKITE_TAG:-}" == v* ]] || [[ "${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then +if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release-* ]] || [[ "${BUILDKITE_TAG:-}" == v* ]] || [[ "${BUILDKITE_PIPELINE_SLUG}" == "julia-publish" ]]; then # First, we have the canonical fully-specified upload target UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}" ) @@ -243,23 +250,60 @@ if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release- fi fi -# If we're a pull request build, upload to a special `-prXXXX` location -if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-pr${BUILDKITE_PULL_REQUEST}-${OS?}-${ARCH?}" ) -fi - # This is the "main" filename that is used. We technically don't need this for uploading, # but it's very convenient for shuttling binaries between buildkite steps. export UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}" +# Staging target: the build step writes the unsigned tarball (write-once, +# directly from the build job) to a commit-sha-gated path in a per-pipeline +# EPHEMERAL bucket (lifecycle-expired; created by ops/terraform). julia-pr +# and julia-ci stage to SEPARATE buckets, and the trusted publish pipeline +# only ever reads the julia-ci bucket -- so a pull-request build can never +# place (or, since paths are write-once, pre-claim) anything the publish +# pipeline would consume. IAM additionally only lets a build write below +# its own commit's staging path within its own pipeline's bucket. +# - For pull requests the pr bucket is the final consumable location +# (juliaup resolves PR number -> head sha via the GitHub API and +# fetches from here). +# - For master/release/tag builds the ci bucket is the input the TRUSTED +# publish pipeline reads, signs, and promotes to the canonical +# UPLOAD_TARGETS. +# The prefix separates the normal / no-GPL artifact families within each +# bucket. +if [[ "${BUILDKITE_PIPELINE_SLUG:-}" == "julia-pr" ]]; then + STAGING_BUCKET="${STAGING_BUCKET:-julialang-ephemeral-pr}" +elif [[ "${BUILDKITE_PIPELINE_SLUG:-}" == julia-buildkite* ]]; then + # The julia-buildkite repository's own self-test CI stages to its own + # bucket, which nothing (juliaup, julia-publish) ever consumes. + STAGING_BUCKET="${STAGING_BUCKET:-julialang-ephemeral-buildkite}" +else + # julia-ci stages here; julia-publish reads its staged artifacts back + STAGING_BUCKET="${STAGING_BUCKET:-julialang-ephemeral-ci}" +fi +export STAGING_BUCKET +# Keyed by BUILDKITE_COMMIT: IAM only allows writing below the +# Buildkite-attested build_commit session tag, and BUILDKITE_COMMIT is +# that same attested value (the pipelines build JuliaLang/julia, so it is +# the julia commit). `git rev-parse HEAD` would normally agree, but the +# attested value is the one that cannot drift from the IAM condition. +# The publish trigger and deploy_docs pass BUILDKITE_COMMIT along, so +# reads agree with what was staged. +export STAGING_TARGET="${STAGING_BUCKET}/${S3_BUCKET_PREFIX}/${BUILDKITE_COMMIT?}/${UPLOAD_FILENAME}" + echo "--- Print the full and short commit hashes" echo "The full commit is: ${LONG_COMMIT}" echo "The short commit is: ${SHORT_COMMIT}" echo "Julia will be installed to: ${JULIA_BINARY}" echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})" echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})" -echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}.tar.gz" -echo "With additional upload targets:" -for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]:1}"; do - echo " -> s3://${UPLOAD_TARGET}.tar.gz" -done +# PR (and other non-master/release/tag) builds have no canonical upload targets: +# they only stage to the ephemeral bucket, so UPLOAD_TARGETS is empty here. +if [[ "${#UPLOAD_TARGETS[@]}" -gt 0 ]]; then + echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}.tar.gz" + echo "With additional upload targets:" + for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]:1}"; do + echo " -> s3://${UPLOAD_TARGET}.tar.gz" + done +else + echo "Staging only (no canonical upload targets): s3://${STAGING_TARGET}.tar.gz" +fi diff --git a/utilities/build_julia.sh b/utilities/build_julia.sh index dee1bc438..d460a76d8 100755 --- a/utilities/build_julia.sh +++ b/utilities/build_julia.sh @@ -12,6 +12,9 @@ source .buildkite/utilities/build_envs.sh # shellcheck source=SCRIPTDIR/word.sh source .buildkite/utilities/word.sh +# Build jobs emit the sysimage from one process, so use the full CPU budget. +export JULIA_IMAGE_THREADS="${JULIA_CPU_THREADS}" + echo "--- Print software versions" uname -a echo @@ -64,9 +67,23 @@ for FLAG in "${MFLAGS[@]}"; do echo " -> ${FLAG}" done +# Stream-filter the build log, rewriting the build dir to [buildroot]. On Windows +# the buildkite agent has no PTY, so this pipe is block-buffered: a long, nearly +# silent step (e.g. the ~25 min libjulia-codegen.dll link) produces no visible +# output until the buffer fills, making the build look hung. GNU sed -u flushes +# per line so the log streams live. (BSD sed on macOS has no -u, and unix jobs run +# under a PTY anyway, so only Windows needs this.) +filter_buildroot() { + if [[ "${TRIPLET}" == *mingw* ]]; then + sed -u "s|$(pwd)|[buildroot]|g" + else + sed "s|$(pwd)|[buildroot]|g" + fi +} + echo "--- Build Julia" echo "Note: The log stream is filtered. [buildroot] replaces pwd $(pwd)" -${MAKE} "${MFLAGS[@]}" 2>&1 | sed "s|$(pwd)|[buildroot]|g" +${MAKE} "${MFLAGS[@]}" 2>&1 | filter_buildroot echo "--- Check that the working directory is clean" @@ -89,7 +106,7 @@ ${JULIA_EXE} -e "import Test; Test.@test Sys.WORD_SIZE == ${EXPECTED_WORD_SIZE:? echo "--- Show build stats" echo "Note: The log stream is filtered. [buildroot] replaces pwd $(pwd)" -${MAKE} "${MFLAGS[@]}" build-stats 2>&1 | sed "s|$(pwd)|[buildroot]|g" +${MAKE} "${MFLAGS[@]}" build-stats 2>&1 | filter_buildroot echo "--- Create build artifacts" ${MAKE} "${MFLAGS[@]}" binary-dist @@ -100,4 +117,35 @@ if [[ "${JULIA_BINARYDIST_FILENAME}.tar.gz" != "${UPLOAD_FILENAME}.tar.gz" ]]; t fi echo "--- Upload build artifacts to buildkite" +# Other jobs in this build (tests, misc checks) consume the tarball as a +# buildkite artifact. buildkite-agent artifact upload "${UPLOAD_FILENAME}.tar.gz" + +echo "--- Stage unsigned tarball to s3://${STAGING_TARGET}.tar.gz" +# Stage straight from the build job (no relay through buildkite artifacts): +# a write-once upload to this pipeline's ephemeral staging bucket, gated by +# this build's commit sha. The untrusted `stage` role can do nothing else. +# shellcheck source=SCRIPTDIR/aws_oidc.sh +source .buildkite/utilities/aws_oidc.sh stage +# shellcheck source=SCRIPTDIR/upload_to_s3.sh +source .buildkite/utilities/upload_to_s3.sh +# The staging buckets disable object ACLs (public read via bucket policy) +UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.tar.gz" "${STAGING_TARGET}.tar.gz" + +# macOS: assemble the Julia.app here and stage it too. The build runs on a Mac, +# so contrib/mac/app's tooling (osacompile, etc.) is available; the trusted +# publish step then only has to codesign + repackage the .app into the signed +# .dmg -- it needs no app-building tools and no Mac. The .app is staged +# UNSIGNED (MACOS_CODESIGN_IDENTITY is unset) under a separate key; the tree +# tarball above is unchanged (test jobs still consume it). +if [[ "${OS}" == "macos" || "${OS}" == "macosnogpl" ]]; then + echo "--- [mac] Assemble the unsigned Julia.app" + # Pass the same MFLAGS as the main build (esp. TAGGED_RELEASE_BANNER): the + # contrib/mac/app rule re-runs binary-dist, and without the matching flags + # build_h.jl regenerates with a different banner, going stale and forcing a + # full system-image rebuild. With them it collapses to a fast re-install+re-tar. + MACOS_CODESIGN_IDENTITY="" ${MAKE} "${MFLAGS[@]}" -C contrib/mac/app "dmg/Julia-${MAJMIN?}.app" + tar zcf "${UPLOAD_FILENAME}.app.tar.gz" -C contrib/mac/app/dmg "Julia-${MAJMIN?}.app" + echo "--- [mac] Stage the unsigned .app to s3://${STAGING_TARGET}.app.tar.gz" + UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.app.tar.gz" "${STAGING_TARGET}.app.tar.gz" +fi diff --git a/utilities/deploy_docs.sh b/utilities/deploy_docs.sh deleted file mode 100644 index a6db09e8c..000000000 --- a/utilities/deploy_docs.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Get the `.buildkite/utilities/docs` folder path -# DOCS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -echo "--- Download built docs" -buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" - -echo "--- Deploy docs" -DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)" -export DOCUMENTER_KEY -echo "Do something here!" diff --git a/utilities/extract_triplet.sh b/utilities/extract_triplet.sh index 7c849ca2a..344b6b23c 100755 --- a/utilities/extract_triplet.sh +++ b/utilities/extract_triplet.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# (Sourced file: deliberately no `set` of shell options here -- they would +# leak into the calling script; strict mode belongs to the entrypoints.) # Extract information from our triplet # Here is an example of a triplet: `x86_64-linux-gnu` diff --git a/utilities/kms_gpg_sign.py b/utilities/kms_gpg_sign.py new file mode 100755 index 000000000..6b02b277a --- /dev/null +++ b/utilities/kms_gpg_sign.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +# This file is a part of Julia. License is MIT: https://julialang.org/license +"""Create OpenPGP (GPG-compatible) detached signatures with AWS KMS. + +This replaces `gpg --detach-sig` for tarball signing. The OpenPGP packet +structure is assembled locally, but the raw RSA signature is produced by +AWS KMS (`kms:Sign` with RSASSA_PKCS1_V1_5_SHA_256). The signing key is +generated inside KMS and never leaves it; the matching OpenPGP public +key (including its self-certification, built with the certificate +helpers below) is exported once with ops/20_export_gpg_pubkey.py and +published as the Julia release signing key. + +The public half of the signing key is read from a normal GPG public key +file (armored or binary) and is only used to derive the issuer +fingerprint/key ID embedded in the signature (and to sanity check the +signature against the modulus). + +AWS credentials are resolved by the AWS CLI; in CI they come from +Buildkite OIDC web identity federation (see utilities/aws_oidc.sh). + +Usage: + kms_gpg_sign.py --public-key tarball_signing.pub.asc \ + --kms-key-id alias/julia-tarball-signing \ + FILE [FILE...] + +Produces FILE.asc for each FILE. + +For testing without KMS access, `--local-key key.pem` signs with a local +RSA private key via openssl, exercising the identical packet construction +path (openssl pkeyutl consumes the same SHA-256 digest KMS would). + +Only RSA signing keys and SHA-256 are supported. +""" + +import argparse +import base64 +import datetime +import hashlib +import os +import struct +import subprocess +import sys +import tempfile +import time + +# OpenPGP constants (RFC 4880) +PKT_SIGNATURE = 2 +PKT_PUBLIC_KEY = 6 +PKT_USER_ID = 13 +SIG_BINARY = 0x00 +SIG_POSITIVE_CERT = 0x13 +ALGO_RSA = 1 # also accept 3 (RSA sign-only); 2 (encrypt-only) is rejected +HASH_SHA256 = 8 +SUBPKT_CREATION_TIME = 2 +SUBPKT_ISSUER_KEY_ID = 16 +SUBPKT_KEY_FLAGS = 27 +SUBPKT_ISSUER_FINGERPRINT = 33 + + +def dearmor(data): + """Extract binary packet data from an ASCII-armored block, or pass through binary.""" + if not data.lstrip().startswith(b"-----BEGIN PGP"): + return data + + lines = data.decode("ascii", "replace").splitlines() + b64 = [] + in_body = False + for line in lines: + line = line.strip() + if line.startswith("-----BEGIN PGP"): + in_body = True + continue + if line.startswith("-----END PGP"): + break + if not in_body: + continue + if not line: + # blank line separates armor headers from body + b64 = [] + continue + if line.startswith("="): + # CRC24 line terminates the body + break + if ":" in line and not b64: + # armor header (Version:, Comment:, ...) + continue + b64.append(line) + + return base64.b64decode("".join(b64)) + + +def iter_packets(data): + """Yield (tag, body) for each OpenPGP packet in data.""" + i = 0 + n = len(data) + while i < n: + ctb = data[i] + if not ctb & 0x80: + raise ValueError(f"invalid packet header byte {ctb:#x} at offset {i}") + i += 1 + if ctb & 0x40: + # New format + tag = ctb & 0x3F + first = data[i] + i += 1 + if first < 192: + length = first + elif first < 224: + length = ((first - 192) << 8) + data[i] + 192 + i += 1 + elif first == 255: + length = struct.unpack(">I", data[i : i + 4])[0] + i += 4 + else: + raise ValueError("partial body lengths not supported") + else: + # Old format + tag = (ctb >> 2) & 0x0F + ltype = ctb & 0x03 + if ltype == 0: + length = data[i] + i += 1 + elif ltype == 1: + length = struct.unpack(">H", data[i : i + 2])[0] + i += 2 + elif ltype == 2: + length = struct.unpack(">I", data[i : i + 4])[0] + i += 4 + else: + raise ValueError("indeterminate length packets not supported") + yield tag, data[i : i + length] + i += length + + +def read_mpi(body, offset): + """Read an OpenPGP MPI; returns (int, new_offset).""" + bits = struct.unpack(">H", body[offset : offset + 2])[0] + nbytes = (bits + 7) // 8 + value = int.from_bytes(body[offset + 2 : offset + 2 + nbytes], "big") + return value, offset + 2 + nbytes + + +def encode_mpi(value): + """Encode an integer as an OpenPGP MPI.""" + bits = value.bit_length() + return struct.pack(">H", bits) + value.to_bytes((bits + 7) // 8, "big") + + +class PublicKeyInfo: + def __init__(self, body): + if body[0] != 4: + raise ValueError(f"only v4 keys supported (got v{body[0]})") + algo = body[5] + if algo not in (1, 3): + raise ValueError(f"only RSA signing keys supported (algorithm {algo})") + self.algorithm = algo + self.n, off = read_mpi(body, 6) + self.e, _ = read_mpi(body, off) + self.fingerprint = hashlib.sha1( + b"\x99" + struct.pack(">H", len(body)) + body + ).digest() + self.key_id = self.fingerprint[-8:] + + +def load_public_key(path): + data = dearmor(open(path, "rb").read()) + for tag, body in iter_packets(data): + if tag == PKT_PUBLIC_KEY: + return PublicKeyInfo(body) + raise ValueError(f"no public key packet found in {path}") + + +def encode_subpacket(sptype, body): + data = bytes([sptype]) + body + if len(data) < 192: + return bytes([len(data)]) + data + if len(data) < 8384: + v = len(data) - 192 + return bytes([192 + (v >> 8), v & 0xFF]) + data + return b"\xff" + struct.pack(">I", len(data)) + data + + +def encode_packet(tag, body): + """Encode a packet with a new-format header.""" + if len(body) < 192: + length = bytes([len(body)]) + elif len(body) < 8384: + v = len(body) - 192 + length = bytes([192 + (v >> 8), v & 0xFF]) + else: + length = b"\xff" + struct.pack(">I", len(body)) + return bytes([0xC0 | tag]) + length + body + + +def crc24(data): + crc = 0xB704CE + for byte in data: + crc ^= byte << 16 + for _ in range(8): + crc <<= 1 + if crc & 0x1000000: + crc ^= 0x1864CFB + return crc & 0xFFFFFF + + +def armor(packet_data, block_type="SIGNATURE"): + b64 = base64.b64encode(packet_data).decode("ascii") + lines = [b64[i : i + 64] for i in range(0, len(b64), 64)] + crc = base64.b64encode(crc24(packet_data).to_bytes(3, "big")).decode("ascii") + return ( + f"-----BEGIN PGP {block_type}-----\n\n" + + "\n".join(lines) + + f"\n={crc}\n-----END PGP {block_type}-----\n" + ) + + +class KmsSigner: + """Sign SHA-256 digests with RSASSA_PKCS1_V1_5_SHA_256 via the AWS CLI.""" + + def __init__(self, key_id): + self.key_id = key_id + + def sign_digest(self, digest): + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + f.write(digest) + digest_path = f.name + try: + out = subprocess.run( + [ + "aws", "kms", "sign", + "--key-id", self.key_id, + "--message", f"fileb://{digest_path}", + "--message-type", "DIGEST", + "--signing-algorithm", "RSASSA_PKCS1_V1_5_SHA_256", + "--output", "text", + "--query", "Signature", + ], + check=True, + capture_output=True, + ) + finally: + os.unlink(digest_path) + return base64.b64decode(out.stdout.strip()) + + +class LocalOpensslSigner: + """Sign SHA-256 digests with a local RSA key via openssl (for testing). + + Uses `openssl pkeyutl` in digest mode, which performs the same + EMSA-PKCS1-v1_5 encoding of a caller-provided digest that KMS does. + """ + + def __init__(self, key_path): + self.key_path = key_path + + def sign_digest(self, digest): + out = subprocess.run( + [ + "openssl", "pkeyutl", "-sign", + "-inkey", self.key_path, + "-pkeyopt", "digest:sha256", + "-pkeyopt", "rsa_padding_mode:pkcs1", + ], + input=digest, + check=True, + capture_output=True, + ) + return out.stdout + + +def build_signature_packet(signer, pubkey, data_hasher, sig_type=SIG_BINARY, + timestamp=None, extra_hashed_subpkts=b""): + """Build a v4 signature packet. + + data_hasher is a hashlib object already updated with the data to be + signed (document for binary sigs; key/uid data for certifications). + """ + if timestamp is None: + timestamp = int(time.time()) + + hashed_subpkts = b"".join([ + encode_subpacket(SUBPKT_ISSUER_FINGERPRINT, b"\x04" + pubkey.fingerprint), + encode_subpacket(SUBPKT_CREATION_TIME, struct.pack(">I", timestamp)), + ]) + extra_hashed_subpkts + unhashed_subpkts = encode_subpacket(SUBPKT_ISSUER_KEY_ID, pubkey.key_id) + + hashed_portion = ( + bytes([4, sig_type, ALGO_RSA, HASH_SHA256]) + + struct.pack(">H", len(hashed_subpkts)) + + hashed_subpkts + ) + trailer = b"\x04\xff" + struct.pack(">I", len(hashed_portion)) + + data_hasher.update(hashed_portion + trailer) + digest = data_hasher.digest() + + raw_signature = signer.sign_digest(digest) + + # Sanity check the raw signature against the public key before publishing: + # s^e mod n must be a valid EMSA-PKCS1-v1_5 encoding ending in our digest. + decoded = pow(int.from_bytes(raw_signature, "big"), pubkey.e, pubkey.n) + decoded_bytes = decoded.to_bytes((pubkey.n.bit_length() + 7) // 8, "big") + if not decoded_bytes.endswith(digest): + raise RuntimeError( + "signature does not verify against the supplied public key; " + "are --public-key and the KMS key the same key pair?" + ) + + body = ( + hashed_portion + + struct.pack(">H", len(unhashed_subpkts)) + + unhashed_subpkts + + digest[:2] + + encode_mpi(int.from_bytes(raw_signature, "big")) + ) + return encode_packet(PKT_SIGNATURE, body) + + +def build_rsa_key_body(n, e, timestamp): + """Build a v4 public key packet body for an RSA key. + + The fingerprint (and thus key ID) covers the creation timestamp, so + the same (n, e, timestamp) always yields the same key identity. + """ + return ( + bytes([4]) + + struct.pack(">I", timestamp) + + bytes([ALGO_RSA]) + + encode_mpi(n) + + encode_mpi(e) + ) + + +# ---- Deriving the public-key identity from the KMS key's public half -------- +# These let the signer reconstruct the OpenPGP public key (and thus the issuer +# fingerprint embedded in signatures) at runtime from `kms:GetPublicKey`, +# instead of reading a committed .asc -- useful for throwaway/test keys where +# publishing a public key block is pointless. ops/20_export_gpg_pubkey.py reuses +# these to build the full published certificate for the production key. + +def parse_der(data, offset=0): + """Parse one DER TLV; returns (tag, value, next_offset).""" + tag = data[offset] + length = data[offset + 1] + offset += 2 + if length & 0x80: + nbytes = length & 0x7F + length = int.from_bytes(data[offset : offset + nbytes], "big") + offset += nbytes + return tag, data[offset : offset + length], offset + length + + +def rsa_components_from_spki(der): + """Extract (n, e) from a SubjectPublicKeyInfo DER blob.""" + _, spki, _ = parse_der(der) # SEQUENCE SubjectPublicKeyInfo + _, _, off = parse_der(spki) # SEQUENCE AlgorithmIdentifier + tag, bits, _ = parse_der(spki, off) # BIT STRING subjectPublicKey + if tag != 0x03 or bits[0] != 0: + raise ValueError("unexpected SPKI structure (not an RSA key?)") + _, rsa, _ = parse_der(bits[1:]) # SEQUENCE RSAPublicKey + tag_n, n_bytes, off = parse_der(rsa) # INTEGER n + tag_e, e_bytes, _ = parse_der(rsa, off) # INTEGER e + if tag_n != 0x02 or tag_e != 0x02: + raise ValueError("unexpected RSAPublicKey structure") + return int.from_bytes(n_bytes, "big"), int.from_bytes(e_bytes, "big") + + +def kms_public_key(key_id): + """Fetch a KMS key's SubjectPublicKeyInfo DER via kms:GetPublicKey.""" + out = subprocess.run( + ["aws", "kms", "get-public-key", "--key-id", key_id, + "--output", "text", "--query", "PublicKey"], + check=True, capture_output=True, + ) + return base64.b64decode(out.stdout.strip()) + + +def local_public_key(key_path): + """SubjectPublicKeyInfo DER for a local RSA private key (testing only).""" + out = subprocess.run( + ["openssl", "rsa", "-in", key_path, "-pubout", "-outform", "DER"], + check=True, capture_output=True, + ) + return out.stdout + + +def kms_key_creation_date(key_id): + """The KMS key's creation time as a unix timestamp (kms:DescribeKey).""" + out = subprocess.run( + ["aws", "kms", "describe-key", "--key-id", key_id, + "--query", "KeyMetadata.CreationDate", "--output", "text"], + check=True, capture_output=True, + ).stdout.decode().strip() + # AWS CLI prints this either as epoch seconds (possibly fractional) or as an + # ISO-8601 string, depending on cli_timestamp_format; accept both. + try: + return int(float(out)) + except ValueError: + return int(datetime.datetime.fromisoformat(out.replace("Z", "+00:00")).timestamp()) + + +def parse_created(value): + """A unix timestamp, or a YYYY-MM-DD date interpreted as UTC midnight.""" + try: + return int(value) + except ValueError: + dt = datetime.datetime.strptime(value, "%Y-%m-%d") + return int(dt.replace(tzinfo=datetime.timezone.utc).timestamp()) + + +def public_key_info_from_kms(key_id, timestamp, local_key=None): + """Reconstruct the PublicKeyInfo (fingerprint, n, e) from the KMS key's + public half plus a pinned creation timestamp -- no committed file.""" + spki = local_public_key(local_key) if local_key else kms_public_key(key_id) + n, e = rsa_components_from_spki(spki) + return PublicKeyInfo(build_rsa_key_body(n, e, timestamp)) + + +def build_certificate(signer, key_body, uid, timestamp): + """Build a minimal transferable public key (certificate): public key + packet + user ID packet + positive self-certification (RFC 4880 + section 5.2.4), signed via signer. Returns (PublicKeyInfo, binary + certificate); wrap with armor(..., "PUBLIC KEY BLOCK") to publish. + """ + pubkey = PublicKeyInfo(key_body) + uid_body = uid.encode() + + hasher = hashlib.sha256() + hasher.update(b"\x99" + struct.pack(">H", len(key_body)) + key_body) + hasher.update(b"\xb4" + struct.pack(">I", len(uid_body)) + uid_body) + + # Key flags: certify + sign + key_flags = encode_subpacket(SUBPKT_KEY_FLAGS, b"\x03") + cert_sig = build_signature_packet( + signer, pubkey, hasher, sig_type=SIG_POSITIVE_CERT, + timestamp=timestamp, extra_hashed_subpkts=key_flags, + ) + + cert = ( + encode_packet(PKT_PUBLIC_KEY, key_body) + + encode_packet(PKT_USER_ID, uid_body) + + cert_sig + ) + return pubkey, cert + + +def sign_file_detached(signer, pubkey, path, output_path=None): + hasher = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + hasher.update(chunk) + + packet = build_signature_packet(signer, pubkey, hasher) + + if output_path is None: + output_path = path + ".asc" + with open(output_path, "w") as f: + f.write(armor(packet)) + return output_path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + pubgroup = parser.add_mutually_exclusive_group(required=True) + pubgroup.add_argument("--public-key", + help="GPG public key file (armored or binary) of the signing key") + pubgroup.add_argument("--public-key-from-kms", action="store_true", + help="derive the signer's OpenPGP public key identity from the " + "KMS key's public half (kms:GetPublicKey) at runtime instead " + "of a committed file; requires --created. Useful for throwaway " + "keys where publishing a public key block is pointless.") + parser.add_argument("--created", + help="override the key creation time (unix timestamp or YYYY-MM-DD " + "UTC) for --public-key-from-kms. Part of the fingerprint; " + "defaults to the KMS key's own CreationDate. Pin it only to " + "match a separately-published pubkey.") + parser.add_argument("--uid", default=None, + help="(unused for signing; accepted for symmetry with the exporter)") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--kms-key-id", + help="AWS KMS key ID/ARN/alias holding the signing key") + group.add_argument("--local-key", + help="local RSA private key PEM (testing only)") + parser.add_argument("files", nargs="+", help="files to sign (creates FILE.asc)") + args = parser.parse_args() + + if args.public_key_from_kms: + # The OpenPGP fingerprint covers the key creation timestamp. Default it + # to the KMS key's own CreationDate (so the identity reflects the real + # key); an explicit --created overrides. A local test key has no KMS + # creation date, so fall back to 0 there. + if args.created: + created = parse_created(args.created) + elif args.local_key: + created = 0 + else: + created = kms_key_creation_date(args.kms_key_id) + pubkey = public_key_info_from_kms( + args.kms_key_id, created, local_key=args.local_key) + else: + pubkey = load_public_key(args.public_key) + print(f"Signing as key {pubkey.fingerprint.hex().upper()}") + + if args.kms_key_id: + signer = KmsSigner(args.kms_key_id) + else: + signer = LocalOpensslSigner(args.local_key) + + for path in args.files: + output = sign_file_detached(signer, pubkey, path) + print(f" {path} -> {output}") + + +if __name__ == "__main__": + main() diff --git a/utilities/macos/build_dmg.sh b/utilities/macos/build_dmg.sh index 88a34e35f..742bdf1c8 100755 --- a/utilities/macos/build_dmg.sh +++ b/utilities/macos/build_dmg.sh @@ -1,78 +1,134 @@ #!/bin/bash +# Build, sign and notarize the Julia .dmg -- on LINUX -- from a PRE-ASSEMBLED, +# already-codesigned Julia.app. +# +# Separation of concerns: the build_ step (on a Mac) assembles the Julia.app +# with all of contrib/mac/app's tooling, and upload_julia.sh codesigns every +# mach-o in it (launcher + bundled julia tree) before this runs. So this step +# only wraps the signed .app in a signed, notarized .dmg -- it needs no +# app-building tooling and no Mac. +# +# Apple-only packaging is replaced as follows (the same approach Mozilla uses to +# package Firefox DMGs on linux): +# hdiutil -> newfs_hfs (the mkfs.hfsplus from hfsprogs/diskdev_cmds) to +# create the HFS+ filesystem, plus the `hfsplus` (populate) and +# `dmg` (compressed UDIF) tools from mozilla/libdmg-hfsplus. +# Code signing and notarization are linux-capable: rcodesign signs with the +# Developer ID key in AWS KMS, and notarization is an App Store Connect API +# call (key also in KMS). set -euo pipefail +THIS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# The already-codesigned Julia.app to package (see upload_julia.sh). +APP_PATH="${APP_PATH:?APP_PATH must point at the (codesigned) Julia.app}" + DMG_PATH="dmg" -mkdir -p "${DMG_PATH}" -APP_PATH="${DMG_PATH}/Julia-${MAJMIN?}.app" DMG_NAME="${UPLOAD_FILENAME?}.dmg" +VOLUME_NAME="Julia-${TAR_VERSION?}" +HFS_IMAGE="$(mktemp -u "${TMPDIR:-/tmp}/julia-dmg-XXXXXX.hfs")" + +# The Developer ID private key lives in AWS KMS; the .dmg itself is signed via +# rcodesign (see utilities/macos/rcodesign/). AWS credentials must already be +# available (source utilities/aws_oidc.sh publish). +MACOS_CODESIGN_KMS_KEY="${MACOS_CODESIGN_KMS_KEY:?}" +NOTARY_API_KEY_FILE="${THIS_DIR}/notary_api_key.json" + +# HFS+/DMG tools. newfs_hfs creates the HFS+ filesystem (the mkfs.hfsplus from +# hfsprogs/diskdev_cmds, built from source in the publish image because hfsprogs +# was dropped from Debian after bullseye); hfsplus (populate) and dmg (convert +# to UDIF) come from libdmg-hfsplus. +MKFSHFS_TOOL="${MKFSHFS_TOOL:-newfs_hfs}" +HFSPLUS_TOOL="${HFSPLUS_TOOL:-hfsplus}" +DMG_TOOL="${DMG_TOOL:-dmg}" + +# Lay out the .dmg contents: the signed .app, the /Applications symlink (added +# below; addall cannot carry it) and the volume icon (taken from inside the .app). +rm -rf "${DMG_PATH}" +mkdir -p "${DMG_PATH}" +cp -aR "${APP_PATH}" "${DMG_PATH}/" +if [[ -f "${APP_PATH}/Contents/Resources/julia.icns" ]]; then + cp "${APP_PATH}/Contents/Resources/julia.icns" "${DMG_PATH}/.VolumeIcon.icns" +fi -# Start by compiling an applescript into a `.app`, which creates the skeleton, which we will fill out -osacompile -o "${APP_PATH}" "contrib/mac/app/startup.applescript" - -# Use `plutil` to fill out the `Info.plist` appropriately -plutil -replace CFBundleDevelopmentRegion -string "en" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleDisplayName -string "Julia" "${APP_PATH}/Contents/Info.plist" -plutil -replace CFBundleIconFile -string "julia.icns" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleIdentifier -string "org.julialang.launcherapp" "${APP_PATH}/Contents/Info.plist" -plutil -replace CFBundleName -string "Julia" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleShortVersionString -string "${MAJMINPAT?}" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleVersion -string "${JULIA_VERSION?}-${SHORT_COMMIT?}" "${APP_PATH}/Contents/Info.plist" -plutil -insert NSHumanReadableCopyright -string "$(date '+%Y') The Julia Project" "${APP_PATH}/Contents/Info.plist" - -# Add icon file for the application and the .dmg -cp "contrib/mac/app/julia.icns" "${APP_PATH}/Contents/Resources/" -cp "contrib/mac/app/julia.icns" "${DMG_PATH}/.VolumeIcon.icns" - -# Add link to `/Applications` -ln -s /Applications "${DMG_PATH}/Applications" - -# Copy our signed tarball into the `.dmg` -cp -aR "${JULIA_INSTALL_DIR?}" "${APP_PATH}/Contents/Resources/julia" - -# Sign the `.app` launcher -.buildkite/utilities/macos/codesign.sh \ - --keychain "${KEYCHAIN_PATH}" \ - --identity "${MACOS_CODESIGN_IDENTITY}" \ - "${APP_PATH}/Contents/MacOS/applet" - -# Create `.dmg`. We create it with 1TB size, but since that is -# a maximum, it has no effect on download or unpack size. -# We define this in a function because we need to do it again later. +# Create the `.dmg`: an HFS+ filesystem image filled with the staged directory, +# converted to a compressed UDIF, then signed. function create_dmg() { - rm -f "${DMG_NAME}" - - hdiutil create \ - "${DMG_NAME}" \ - -size 1t \ - -fs HFS+ \ - -volname "Julia-${TAR_VERSION?}" \ - -imagekey zlib-level=9 \ - -srcfolder "${DMG_PATH}" + rm -f "${DMG_NAME}" "${HFS_IMAGE}" + + # Size the filesystem just above the measured tree size. The margin covers + # HFS+ catalog/extents/bitmap metadata plus per-file allocation rounding over + # the thousands of files in the Julia tree; `du -sm` already rounds each file + # up to a 4 KB block, so the real overhead is only ~5% and a 10% + 64 MB + # margin sits comfortably above the empirical failure floor (under-sizing + # fails loudly mid-populate with "rawFileWrite ... allocate", never a corrupt + # dmg). Keep it TIGHT: libdmg-hfsplus's `addall` cost scales with the image's + # total block count, not the file data -- every allocated block is zeroed and + # its bitmap byte seeked individually -- so the old 1.5x+256 MB of free space + # made the populate ~3x slower for no benefit (the final UDIF is compressed). + local size_mb + size_mb="$(( $(du -sm "${DMG_PATH}" | cut -f1) * 11 / 10 + 64 ))" + truncate -s "${size_mb}M" "${HFS_IMAGE}" + "${MKFSHFS_TOOL}" -v "${VOLUME_NAME}" "${HFS_IMAGE}" + + # `addall` logs every file and directory it copies (thousands of lines), which + # both floods the build log and slows the step (all of it streams to the + # agent). Capture it and only surface the tail on failure. + # + # `--symlinks clone_link` is REQUIRED: the Julia tree ships ~60 versioned + # dylib symlinks (libjulia.dylib -> libjulia.1.14.0.dylib, etc.). The default + # policy dereferences them into regular files, which breaks the codesign + # symlink seals (rcodesign seals them as {symlink: target}); Apple's notary + # then reports every one as "file modified" and rejects the bundle's main + # executable as having an invalid signature. clone_link recreates them as + # real HFS+ symlinks so the seals match. + local addall_log + addall_log="$(mktemp)" + if ! "${HFSPLUS_TOOL}" "${HFS_IMAGE}" addall "${DMG_PATH}" --symlinks clone_link > "${addall_log}" 2>&1; then + echo "ERROR: hfsplus addall failed; tail of its output:" >&2 + tail -30 "${addall_log}" >&2 + rm -f "${addall_log}" + return 1 + fi + rm -f "${addall_log}" + "${HFSPLUS_TOOL}" "${HFS_IMAGE}" symlink "/Applications" "/Applications" + # Mark the volume root as having a custom icon (.VolumeIcon.icns) + "${HFSPLUS_TOOL}" "${HFS_IMAGE}" attr "/" C + + "${DMG_TOOL}" build "${HFS_IMAGE}" "${DMG_NAME}" + rm -f "${HFS_IMAGE}" # Sign the `.dmg` itself - .buildkite/utilities/macos/codesign.sh \ - --keychain "${KEYCHAIN_PATH}" \ - --identity "${MACOS_CODESIGN_IDENTITY}" \ + "${THIS_DIR}/codesign.sh" \ + --kms-key "${MACOS_CODESIGN_KMS_KEY}" \ "${DMG_NAME}" } create_dmg -# Upload the `.dmg` for notarization - -xcrun notarytool \ - submit \ - --apple-id "${NOTARIZATION_APPLE_ID}" \ - --password "${NOTARIZATION_APPLE_KEY}" \ - --team-id "A427R7F42H" \ - --wait \ - "${DMG_NAME}" - -# Staple the notarization to the app -xcrun stapler staple "${APP_PATH}" +# Notarize the `.dmg`, then staple the ticket onto the `.dmg` itself. The App +# Store Connect API key also lives in KMS; notary_api_key.json contains no secret +# material (see ops/21_import_notary_key.sh). +# +# The non-production publish test stack sets PUBLISH_SKIP_NOTARIZATION=1: +# notarization is a hosted Apple round-trip with no self-signable equivalent, so +# the test pipeline skips it. The .dmg is still KMS-signed; only the Apple +# notarize + staple are skipped. +if [[ "${PUBLISH_SKIP_NOTARIZATION:-0}" != "1" ]]; then + RCODESIGN="$("${THIS_DIR}/get_rcodesign.sh")" + + "${RCODESIGN}" notary-submit \ + --api-key-file "${NOTARY_API_KEY_FILE}" \ + --wait \ + "${DMG_NAME}" -# Re-build the .dmg from the app now that it's notarized -create_dmg + # Staple the ticket onto the .dmg itself -- no rebuild needed. The stapled + # .dmg validates offline on download; Gatekeeper then assesses the .app when + # it is first launched from the mounted volume. + "${RCODESIGN}" staple "${DMG_NAME}" +else + echo "Skipping notarization (PUBLISH_SKIP_NOTARIZATION=1): .dmg is KMS-signed but not notarized/stapled." >&2 +fi # Cleanup things we created here rm -rf "${DMG_PATH}" diff --git a/utilities/macos/codesign.sh b/utilities/macos/codesign.sh index f57c9bc65..7b99eea83 100755 --- a/utilities/macos/codesign.sh +++ b/utilities/macos/codesign.sh @@ -4,107 +4,175 @@ set -euo pipefail usage() { - echo "Usage: $0 [--identity=] [--keychain=] " + echo "Usage: $0 [--kms-key=] [--certificate=] " echo echo "Parameter descriptions:" echo - echo " id: A codesigning identity hash or '-' to denote ad-hoc signing" - echo " Default value is '-'" + echo " kms-key: AWS KMS key ID/ARN/alias holding the Developer ID private key." + echo " If not given, performs ad-hoc signing with the system codesign." echo - echo " keychain_path: A path to a keychain file that contains the given identity" - echo " If not given, uses the default keychain search path." + echo " certificate: Path to the Developer ID certificate (PEM) paired with the" + echo " KMS key. Defaults to developer_id.pem next to this script." + echo + echo " For a .app directory the default is a two-phase sign: first every nested" + echo " Mach-O individually, then a bundle seal. These flags split the phases so a" + echo " caller can interpose work (e.g. patching stdlib pkgimage checksums, which" + echo " must read the *signed* pkgimages yet be sealed into CodeResources):" + echo " --no-seal: sign the nested Mach-Os but do NOT seal the bundle." + echo " --seal-only: skip the nested pass; only seal the bundle." echo echo " target: A file or directory to codesign (must come last!)" + echo + echo "When a KMS key is used, signing is performed with rcodesign (apple-codesign" + echo "with the AWS KMS backend; see utilities/macos/rcodesign/). AWS credentials" + echo "must be available, e.g. via 'source utilities/aws_oidc.sh publish'." } -abspath() { - echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")" -} +THIS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" if [ "$#" -lt 1 ]; then usage exit 1 fi -# Default codesign identity to ad-hoc signing -CODESIGN_IDENTITY="-" +KMS_KEY="" +# The Developer ID certificate (PEM) paired with the KMS key. Defaults to the +# production developer_id.pem next to this script; the non-production publish +# test stack overrides it (a self-signed cert for the test KMS key) via +# MACOS_CODESIGN_CERT. An explicit --certificate still wins over both. +CERTIFICATE="${MACOS_CODESIGN_CERT:-${THIS_DIR}/developer_id.pem}" + +# Bundle signing phases (see usage): by default a .app target gets both the +# per-file nested pass and the final bundle seal. --no-seal / --seal-only run +# only one phase so a caller can interpose work between them. +DO_NESTED=1 +DO_SEAL=1 while [ "$#" -gt 1 ]; do case "${1}" in - --identity) - CODESIGN_IDENTITY="$2" - shift - shift - ;; - --identity=*) - CODESIGN_IDENTITY="${1#*=}" - shift - ;; - --keychain) - KEYCHAIN_PATH="$2" - shift - shift - ;; - --keychain=*) - KEYCHAIN_PATH="${1#*=}" - shift - ;; + --kms-key) + KMS_KEY="$2"; shift; shift ;; + --kms-key=*) + KMS_KEY="${1#*=}"; shift ;; + --certificate) + CERTIFICATE="$2"; shift; shift ;; + --certificate=*) + CERTIFICATE="${1#*=}"; shift ;; + --no-seal) + DO_SEAL=0; shift ;; + --seal-only) + DO_NESTED=0; shift ;; *) echo "Unknown argument '$1'" usage - exit 1 - ;; + exit 1 ;; esac done +TARGET="${1}" -# Verify keychain arg -KEYCHAIN_ARGS=() -if [ -n "${KEYCHAIN_PATH:-}" ]; then - # Expand KEYCHAIN_PATH, in case it contains `~`, - # also ensuring that it is an absolute path. - KEYCHAIN_PATH="$(abspath "${KEYCHAIN_PATH}")" - if [ -f "${KEYCHAIN_PATH}" ]; then - KEYCHAIN_ARGS+=("--keychain" "${KEYCHAIN_PATH}") - - # Ensure that the given keychain has already been added to our search list and is unlocked - if ! security show-keychain-info "${KEYCHAIN_PATH}" >/dev/null 2>/dev/null; then - echo "Must unlock keychain '${KEYCHAIN_PATH}' first!" >&2 - exit 1 - fi - else - echo "Given keychain file '${KEYCHAIN_PATH}' not found!" >&2 - exit 1 - fi +if [ "${DO_NESTED}" = 0 ] && [ "${DO_SEAL}" = 0 ]; then + echo "ERROR: --no-seal and --seal-only are mutually exclusive" >&2 + exit 1 fi -# Verify that we can load the given identity -if [ "${CODESIGN_IDENTITY}" != "-" ]; then - if ! security find-identity -p codesigning "${KEYCHAIN_PATH:-}" 2>&1 | grep "${CODESIGN_IDENTITY}" >/dev/null; then - echo "security find-identity found no matching identity for '${CODESIGN_IDENTITY}':" - security find-identity -p codesigning "${KEYCHAIN_PATH:-}" - exit 1 - fi -fi +# True if $1 begins with a Mach-O magic (thin LE/BE 32/64-bit, or a fat/universal +# archive). rcodesign signs Mach-O binaries (and bundles/dmgs) only; the bundle +# tree also holds executable *non*-Mach-O files (scripts, .jl, .tbd, Makefiles +# with the +x bit) that `find -perm -0111` matches. rcodesign rejects those with +# "specified path is not of a recognized type"; they are plain data and need no +# code signature, so we skip them (see the directory loop below). +is_macho() { + local magic + magic="$(od -An -tx1 -N4 -- "${1}" 2>/dev/null | tr -d ' \n')" + case "${magic}" in + # MH_MAGIC/CIGAM (32), MH_MAGIC_64/CIGAM_64, FAT_MAGIC/CIGAM (+_64) + feedface|cefaedfe|feedfacf|cffaedfe|cafebabe|bebafeca|cafebabf|bfbafeca) return 0 ;; + *) return 1 ;; + esac +} -do_codesign() { - # Codesign with the given identity, opting into the hardened runtime, - # applying the entitlements plist we maintain within this repository, - # asking for timestamping, performing "deep" codesigning for bundles, - # and replacing any previous signatures that may exist. - codesign --sign "${CODESIGN_IDENTITY}" \ +do_adhoc_codesign() { + # Ad-hoc signing needs no key; use the system codesign as before. + codesign --sign "-" \ --option=runtime \ - --entitlements "$(dirname "${0}")/Entitlements.plist" \ - "${KEYCHAIN_ARGS[@]}" \ + --entitlements "${THIS_DIR}/Entitlements.plist" \ --timestamp \ --force \ "${1}" } -if [ -f "${1}" ]; then +do_kms_codesign() { + # Sign with the Developer ID key in AWS KMS via rcodesign. The hardened + # runtime flag and entitlements match what we passed to Apple codesign. + # rcodesign prints ~10 chatty lines per file; across hundreds of files signed + # in parallel that floods (and interleaves in) the log, so capture the output + # and only emit it if the signature actually fails. + # + # A directory target is the .app bundle: seal it WITHOUT re-signing nested + # code (--shallow). The per-file pass already signs every nested Mach-O + # individually; Apple recommends against re-signing nested code via a deep + # bundle sign, and shallow mode also avoids touching unsignable members + # (static archives, dSYM DWARF) -- they are sealed as resources. + local shallow_args=() + [[ -d "${1}" ]] && shallow_args=( --shallow ) + local out + if ! out="$("${RCODESIGN_BIN}" sign \ + --aws-kms-key "${KMS_KEY}" \ + --aws-kms-certificate-file "${CERTIFICATE}" \ + --code-signature-flags runtime \ + --entitlements-xml-file "${THIS_DIR}/Entitlements.plist" \ + ${shallow_args[@]+"${shallow_args[@]}"} \ + "${1}" 2>&1)"; then + echo "ERROR: rcodesign failed for ${1}:" >&2 + echo "${out}" >&2 + return 1 + fi +} + +if [ -n "${KMS_KEY}" ]; then + if [ ! -f "${CERTIFICATE}" ]; then + echo "ERROR: certificate '${CERTIFICATE}' not found!" >&2 + exit 1 + fi + RCODESIGN_BIN="$("${THIS_DIR}/get_rcodesign.sh")" + + # Each rcodesign process resolves AWS credentials independently. Signing a + # whole bundle means hundreds of processes, and under Buildkite OIDC each + # would run its own STS AssumeRoleWithWebIdentity. Even a small bounded pool + # overwhelmed the (proxied) STS endpoint enough that ~8% of resolutions + # exceeded the AWS SDK's 5s budget ("identity resolver timed out after 5s") + # and the signature failed. Assume the role ONCE here and hand every signer + # the resulting static session credentials via the SDK's first, network-free + # environment provider -- credential resolution becomes a local lookup that + # cannot time out, no matter how many signers run concurrently. (Only when + # running under an OIDC web-identity token; a local run with static creds + # already in the environment keeps them.) + if [[ -z "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_WEB_IDENTITY_TOKEN_FILE:-}" ]]; then + echo "Resolving shared AWS session credentials for rcodesign (one STS call for all signers)" + _kms_creds="$(aws sts assume-role-with-web-identity \ + --role-arn "${AWS_ROLE_ARN:?}" \ + --role-session-name "${AWS_ROLE_SESSION_NAME:-rcodesign}" \ + --web-identity-token "$(cat "${AWS_WEB_IDENTITY_TOKEN_FILE}")" \ + --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \ + --output text)" + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"${_kms_creds}" + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + unset _kms_creds + fi + + do_codesign() { do_kms_codesign "$@"; } + IDENTITY_DESC="KMS key ${KMS_KEY}" +else + do_codesign() { do_adhoc_codesign "$@"; } + IDENTITY_DESC="ad-hoc identity" +fi + +if [ -f "${TARGET}" ]; then # If we're codesigning a single file, directly invoke codesign on that file - echo "Codesigning file ${1} with identity ${CODESIGN_IDENTITY}" - do_codesign "${1}" -elif [ -d "${1}" ]; then + echo "Codesigning file ${TARGET} with ${IDENTITY_DESC}" + do_codesign "${TARGET}" +elif [ -d "${TARGET}" ]; then + if [ "${DO_NESTED}" = 1 ]; then # Create a fifo to communicate from `find` to `while` trap 'rm -rf $TMPFIFODIR' EXIT TMPFIFODIR="$(mktemp -d)" @@ -113,21 +181,72 @@ elif [ -d "${1}" ]; then # If we're codesigning a whole directory, use `find` to discover every # executable file within the directory, then pass that off to a while # read loop. This safely handles whitespace in filenames. - find "${1}" -type f -perm -0111 -print0 > "$TMPFIFODIR/findpipe" & + find "${TARGET}" -type f -perm -0111 -print0 > "$TMPFIFODIR/findpipe" & - # This while loop reads in from the fifo, and invokes `do_codesign`, - # but it does so in a background task, so that the codesigning can - # happen in parallel. This speeds things up by a few seconds. - echo "Codesigning dir ${1} with identity ${CODESIGN_IDENTITY}" + # Sign the discovered files with a BOUNDED pool of background workers so the + # codesigning happens in parallel without an unbounded fan-out. A full bundle + # (e.g. the Julia tree) holds hundreds of mach-o files; backgrounding a + # signer for *every* one at once spawned hundreds of simultaneous rcodesign + # processes, each independently assuming the KMS-signing OIDC role via STS + # and holding KMS / Apple-timestamp connections open for the duration of the + # signature. That much concurrent pressure pushed some credential + # resolutions past the AWS SDK's 5s budget ("identity resolver timed out + # after 5s"), failing the signature. Cap the concurrency (MACOS_CODESIGN_JOBS). + MAX_JOBS="${MACOS_CODESIGN_JOBS:-8}" + echo "Codesigning dir ${TARGET} with ${IDENTITY_DESC} (up to ${MAX_JOBS} concurrent)" + # A non-empty STATUS_DIR after the run means at least one signer failed: the + # trailing `wait` returns 0 regardless, so failures are recorded out-of-band + # rather than silently swallowed (never ship a partially-signed bundle). + STATUS_DIR="$(mktemp -d)" NUM_CODESIGNS=0 + NUM_SKIPPED=0 while IFS= read -r -d '' exe_file; do - do_codesign "${exe_file}" & + # Skip files the KMS signer (rcodesign) can't sign -- they need no + # signature, and the ad-hoc `codesign` handles them, so only filter on + # the KMS path: + # - non-Mach-O (scripts, .jl, .tbd, Makefiles, ...): rcodesign rejects + # them as "specified path is not of a recognized type". + # - static archives (*.a): a universal/"fat" archive like + # libclang_rt.osx.a carries a fat magic so it passes is_macho, but it + # is an `ar` archive, not a signable binary -- rcodesign panics + # parsing it ("index out of bounds"). Static libs are link-time input. + if [ -n "${KMS_KEY}" ] && { [[ "${exe_file}" == *.a ]] || ! is_macho "${exe_file}"; }; then + echo "Skipping non-signable file: ${exe_file}" + NUM_SKIPPED="$((NUM_SKIPPED + 1))" + continue + fi + # Block until a worker slot frees up before launching the next signer. + while (( "$(jobs -rp | wc -l)" >= MAX_JOBS )); do wait -n 2>/dev/null || true; done + { do_codesign "${exe_file}" || touch "${STATUS_DIR}/fail.${NUM_CODESIGNS}"; } & NUM_CODESIGNS="$((NUM_CODESIGNS + 1))" done < "${TMPFIFODIR}/findpipe" wait - echo "Codesigned ${NUM_CODESIGNS} files" + + NUM_FAILED="$(find "${STATUS_DIR}" -type f | wc -l)" + rm -rf "${STATUS_DIR}" + if [ "${NUM_FAILED}" -ne 0 ]; then + echo "ERROR: ${NUM_FAILED} of ${NUM_CODESIGNS} codesign operations failed" >&2 + exit 1 + fi + echo "Codesigned ${NUM_CODESIGNS} files (skipped ${NUM_SKIPPED} non-Mach-O)" + fi + + # A .app is a bundle: its main executable (Contents/MacOS/applet) must be + # signed in bundle context -- sealing Contents/_CodeSignature/CodeResources + # and binding Info.plist -- or Apple's notary rejects its signature as + # invalid. The per-file pass above signs every nested Mach-O (so the notary + # sees them all signed); this final pass seals the bundle and re-signs the + # main executable correctly. rcodesign detects the bundle from the path. + # + # When split across phases (--no-seal / --seal-only) the caller patches the + # stdlib pkgimage checksums between the nested pass and this seal, so the + # sealed CodeResources covers the final (checksum-corrected) .ji files. + if [ "${DO_SEAL}" = 1 ] && [[ "${TARGET}" == *.app && -d "${TARGET}/Contents" ]]; then + echo "Sealing .app bundle ${TARGET} with ${IDENTITY_DESC}" + do_codesign "${TARGET}" + fi else - echo "Given codesigning target '${1}' not a file or directory!" >&2 + echo "Given codesigning target '${TARGET}' not a file or directory!" >&2 usage exit 1 fi diff --git a/utilities/macos/developer_id.pem b/utilities/macos/developer_id.pem new file mode 100644 index 000000000..8a3e08d77 --- /dev/null +++ b/utilities/macos/developer_id.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF0jCCBLqgAwIBAgIQYLifKh99DyZM1MtUkk4lNjANBgkqhkiG9w0BAQsFADBe +MS0wKwYDVQQDDCREZXZlbG9wZXIgSUQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkx +CzAJBgNVBAsMAkcyMRMwEQYDVQQKDApBcHBsZSBJbmMuMQswCQYDVQQGEwJVUzAe +Fw0yNjA2MTUxODI3MzZaFw0zMTA2MTYxODI3MzVaMIGhMRowGAYKCZImiZPyLGQB +AQwKQTQyN1I3RjQySDFDMEEGA1UEAww6RGV2ZWxvcGVyIElEIEFwcGxpY2F0aW9u +OiBKdWxpYSBDb21wdXRpbmcgTExDIChBNDI3UjdGNDJIKTETMBEGA1UECwwKQTQy +N1I3RjQySDEcMBoGA1UECgwTSnVsaWEgQ29tcHV0aW5nIExMQzELMAkGA1UEBhMC +VVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKPgRbsUPdnajWEGYN +K8ip3wJuty9utiOSQwzEJTNpgQH1bZY0ITqnHYsDs+uifmUQO/eIcEvNLttJoBTX +u5xjIJr7dJqOiyeUdjdG8+Af3ahgLJ7/YgMI5BfDg54VYpvpGfQb9LdyFyhKcowx +aUBkk7Rn2g8NN1XpWFJNF2mZC35JYjUUoSDTfvuc55ELlKu/qsZ9bBK6rxIzzgVE +uSvvKkyyFxhjEvaYNq+st6nbeuJUatTehzJvbRsxAxPn1vMWnBa8iv5K3FeYyXmh +TJhGIKQzni0oZvXxvmj46ar3refzGNrrUT/4Rd8w5c7v1pg99km8t/Mvtz9PX6fl ++Q79AgMBAAGjggJGMIICQjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFPg6DGkR +duDtrNHrpln6N9XEVbAeMHIGCCsGAQUFBwEBBGYwZDAuBggrBgEFBQcwAoYiaHR0 +cDovL2NlcnRzLmFwcGxlLmNvbS9kZXZpZGcyLmRlcjAyBggrBgEFBQcwAYYmaHR0 +cDovL29jc3AuYXBwbGUuY29tL29jc3AwMy1kZXZpZGcyMDEwggEeBgNVHSAEggEV +MIIBETCCAQ0GCSqGSIb3Y2QFATCB/zCBwwYIKwYBBQUHAgIwgbYMgbNSZWxpYW5j +ZSBvbiB0aGlzIGNlcnRpZmljYXRlIGJ5IGFueSBwYXJ0eSBhc3N1bWVzIGFjY2Vw +dGFuY2Ugb2YgdGhlIHRoZW4gYXBwbGljYWJsZSBzdGFuZGFyZCB0ZXJtcyBhbmQg +Y29uZGl0aW9ucyBvZiB1c2UsIGNlcnRpZmljYXRlIHBvbGljeSBhbmQgY2VydGlm +aWNhdGlvbiBwcmFjdGljZSBzdGF0ZW1lbnRzLjA3BggrBgEFBQcCARYraHR0cHM6 +Ly93d3cuYXBwbGUuY29tL2NlcnRpZmljYXRlYXV0aG9yaXR5LzAWBgNVHSUBAf8E +DDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUHppTZm/VSGqN2etBWOQw0ReNk9owDgYD +VR0PAQH/BAQDAgeAMB8GCiqGSIb3Y2QGASEEEQwPMjAxNDA4MTEwMDAwMDBaMBMG +CiqGSIb3Y2QGAQ0BAf8EAgUAMA0GCSqGSIb3DQEBCwUAA4IBAQB1hLlmd92DFBm/ +JI1m/nfy4IQEJhZaakEV0RVRVo+JIuNWVuaD2gAukkV3dHj5L6Ju9yRyUPja7WAz +6S/e5p2LvYzosKbxNN4UxSmSNeaXMHObTCQfacv/sgiK9FN9XjcHOBKfiRiQt139 +raiwdfqRqJMaUWki+qi76xnD7ZrZTcQKk4Xz7zLXTTuWBSq62xkwqb6o+zqfwWuJ +5otNJGR1zockRUUyOa9TYw7yuFsHRA6XICBF2Y5MYULD8XboNjXoU41qONGWPXW6 +s2fFZuHR+Ix15sbNKxg5kLjxR5yPNU6asANIB1IIp3YU1gYCup592jdPjuoYDQlT +F6VAMqDL +-----END CERTIFICATE----- diff --git a/utilities/macos/developer_id_test.pem b/utilities/macos/developer_id_test.pem new file mode 100644 index 000000000..18bd6d944 --- /dev/null +++ b/utilities/macos/developer_id_test.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkDCCAnigAwIBAgIUfvASH5xsMkjkv73VJ+5sHrJvZeMwDQYJKoZIhvcNAQEL +BQAwaTE3MDUGA1UEAwwuSnVsaWEgVEVTVCBjb2Rlc2lnbmluZyBDQSAoTk9UIEZP +UiBQUk9EVUNUSU9OKTEhMB8GA1UECgwYVGhlIEp1bGlhIFByb2plY3QgKFRFU1Qp +MQswCQYDVQQGEwJVUzAeFw0yNjA2MTQxODI1MjNaFw0zNjA2MTExODI1MjNaMCEx +HzAdBgNVBAMMFkFwcGxlIENvZGUgU2lnbmluZyBDU1IwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCwCwNYOVkJQHu2MYPu9DAmc8fI3fOLRK13WQLHwgUL +unqxBroDTWbYgU0X5CJlWN9NYNCovbRPw9zPPPUIc8SZNSpeGVNKphI9H2dFLVMh +uX0W8exFFddocZs6sXrp2+CCtHVXZbn5C3LSVjz9PqMKxwV2926YyQdmzVQezhMY +hjjfOg8mKAWA5RV85xukccmUHio2RxrlnGcRfb8j01ielSSzebCNCxy7Z8ktv7g4 +DoLQZW4mpBxWV2Z2hObefpouYAD0V0S5dkO+11hhNXOX4voPRneJ1pASHPPmwAyA +6mFJB2Tn4omjC1Lc0eFTxecSLj3R+S23mGrgWCkrHc9BAgMBAAGjeDB2MA4GA1Ud +DwEB/wQEAwIHgDAWBgNVHSUBAf8EDDAKBggrBgEFBQcDAzAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTB/iMvz8TydFfOegZfq5H2H0Z9yDAfBgNVHSMEGDAWgBS4WNvN +cq8++YoBFPV7ZhDMMjoSojANBgkqhkiG9w0BAQsFAAOCAQEAP40ZEZC89YVHZks+ +rmQAdLMYW0mgL3A4mdYOCZN7NLOHgfORnS4rKSU4q0q7VRRxypny7u5raR4u9IO3 +4rW5UwUuB0SuysBp8cf6kXqCSyrqnKhthNhPGRqBFEpxDH+yCmH2Ke3zSLHC6bx+ +7f7b8f7qm1s8bBTiqwm43h5Xg+kX/NiWEduWSsLVT37mzAMtRY8wvuWF3JfMlMaD +Ol++5+7W5VrVih3SJlLpgGBTXcTy8Wpe6FlOuI4m600WAH8j7gz6ktI+GGDAhMbk +071/7zGlxOYIb+AmbBmr4GR7JyrXVWhQSL8LJpYRg981nlLULfJXq63nph3RP+/A +Jlxx5A== +-----END CERTIFICATE----- diff --git a/utilities/macos/get_rcodesign.sh b/utilities/macos/get_rcodesign.sh new file mode 100755 index 000000000..4c37e78ed --- /dev/null +++ b/utilities/macos/get_rcodesign.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Fetch (and cache) the pinned rcodesign binary; prints its path. +# +# rcodesign is apple-codesign with the AWS KMS signing backend, built from +# JuliaCI/apple-platform-rs by Yggdrasil (R/rcodesign) and published as a +# rcodesign_jll release asset (see utilities/macos/rcodesign/README.md). +# The sha256 below pins the exact tarball. It runs on the LINUX publish +# agent: rcodesign signs, notarizes and staples Apple artifacts +# cross-platform. +# +# Set RCODESIGN to override (e.g. a locally built binary). +set -euo pipefail + +RCODESIGN_VERSION="0.29.0+1" +RCODESIGN_CRATE_VERSION="0.29.0" +RCODESIGN_SHA256_x86_64_linux_gnu="9756d1cf93358e3bfcd12457450950d538c858325550cb8589f397beffbbaa06" +RCODESIGN_BASE_URL="${RCODESIGN_BASE_URL:-https://github.com/JuliaBinaryWrappers/rcodesign_jll.jl/releases/download/rcodesign-v0.29.0%2B1}" + +if [ -n "${RCODESIGN:-}" ]; then + echo "${RCODESIGN}" + exit 0 +fi + +case "$(uname -sm)" in + "Linux x86_64") TRIPLET="x86_64-linux-gnu" ;; + *) + echo "ERROR: no pinned rcodesign build for '$(uname -sm)'; set RCODESIGN" >&2 + exit 1 + ;; +esac + +SHA_VAR="RCODESIGN_SHA256_${TRIPLET//-/_}" +EXPECTED_SHA="${!SHA_VAR}" +CACHE_DIR="${HOME}/.cache/julia-buildkite" +BINARY="${CACHE_DIR}/rcodesign-${RCODESIGN_VERSION}-${TRIPLET}" + +if [ ! -x "${BINARY}" ]; then + mkdir -p "${CACHE_DIR}" + TARBALL="${BINARY}.tar.gz.tmp" + curl --fail -sSL -o "${TARBALL}" \ + "${RCODESIGN_BASE_URL}/rcodesign.v${RCODESIGN_CRATE_VERSION}.${TRIPLET}.tar.gz" >&2 + ACTUAL_SHA="$(sha256sum "${TARBALL}" | cut -d' ' -f1)" + if [ "${ACTUAL_SHA}" != "${EXPECTED_SHA}" ]; then + echo "ERROR: rcodesign sha256 mismatch (got ${ACTUAL_SHA}, expected ${EXPECTED_SHA})" >&2 + rm -f "${TARBALL}" + exit 1 + fi + EXTRACT_DIR="$(mktemp -d "${CACHE_DIR}/rcodesign-extract-XXXXXX")" + tar -xzf "${TARBALL}" -C "${EXTRACT_DIR}" bin/rcodesign + mv "${EXTRACT_DIR}/bin/rcodesign" "${BINARY}" + rm -rf "${EXTRACT_DIR}" "${TARBALL}" +fi + +echo "${BINARY}" diff --git a/utilities/macos/notary_api_key.json b/utilities/macos/notary_api_key.json new file mode 100644 index 000000000..612f5043e --- /dev/null +++ b/utilities/macos/notary_api_key.json @@ -0,0 +1,6 @@ +{ + "issuer_id": "69a6de82-7ea6-47e3-e053-5b8c7c11a4d1", + "key_id": "V8W5RG5668", + "aws_kms_key": "arn:aws:kms:us-east-1:873569884612:key/3a9c20f8-f34d-447f-b08d-043d863b7a3e", + "aws_kms_region": "us-east-1" +} diff --git a/utilities/macos/rcodesign/README.md b/utilities/macos/rcodesign/README.md new file mode 100644 index 000000000..fc49ee507 --- /dev/null +++ b/utilities/macos/rcodesign/README.md @@ -0,0 +1,48 @@ +# rcodesign with AWS KMS backend + +This directory documents the tooling for macOS codesigning + notarization +without secrets on the build agents. We use +[apple-codesign](https://github.com/indygreg/apple-platform-rs/tree/main/apple-codesign) +(`rcodesign`) with an AWS KMS signing backend, maintained on +[`JuliaCI/apple-platform-rs`](https://github.com/JuliaCI/apple-platform-rs/tree/julia-build) +(branch `julia-build`: upstream main plus the patch series listed under +Upstreaming below). + +## How it works + +* The **Developer ID Application certificate's private key** is an RSA-2048 + key held in AWS KMS (`SIGN_VERIFY`). `rcodesign sign --aws-kms-key + --aws-kms-certificate-file ` performs each low-level signature via + the KMS `Sign` API. Key material never reaches the agent. +* The **App Store Connect API key** (ECDSA P-256, used to mint ES256 JWTs + for the notary service) is also held in KMS. The unified API key JSON + (`utilities/macos/notary_api_key.json`) references the KMS key ARN and + contains **no secret material**, so it is committed in plaintext. +* AWS credentials come from Buildkite OIDC via + `AWS_WEB_IDENTITY_TOKEN_FILE`/`AWS_ROLE_ARN` (see + `utilities/aws_oidc.sh`); IAM policies restrict `kms:Sign` to the + appropriate pipeline/step session tags. + +## Building + +Yggdrasil builds the binary (recipe `R/rcodesign`, `GitSource` pinned to a +`julia-build` commit, `--features aws-kms` on all platforms) and publishes it +as [`rcodesign_jll`](https://github.com/JuliaBinaryWrappers/rcodesign_jll.jl) +release assets. CI fetches and sha256-verifies the pinned tarball via +`utilities/macos/get_rcodesign.sh`. + +To ship a new build: push the updated `julia-build` branch to +`JuliaCI/apple-platform-rs`, open a Yggdrasil PR bumping the `GitSource` +commit in `R/rcodesign/build_tarballs.jl`, and once the new `rcodesign_jll` +release exists, update the version / sha256 / base URL pins in +`get_rcodesign.sh`. + +## Upstreaming + +The patches are submitted to `indygreg/apple-platform-rs` as three series +(branches on `KenoAIStaging/apple-platform-rs`): `upstream-macho-robustness` +(zero-slice Mach-O fixes), `upstream-asc-signer` (external ES256 signers for +App Store Connect tokens), and `upstream-aws-kms` (the feature-gated +`aws-kms` backend, docs in `apple_codesign_aws_kms.rst`, unit tests). Once +they land and a release is cut, the Yggdrasil recipe returns to plain +upstream release tarballs. diff --git a/utilities/publish.sh b/utilities/publish.sh new file mode 100755 index 000000000..87de36699 --- /dev/null +++ b/utilities/publish.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Single trusted publish step. +# +# Runs once in the julia-publish pipeline and promotes every staged build +# to its final release location: it verifies the release commit, assumes the +# trusted `publish` role once, then iterates over every triplet, signing +# (macOS via rcodesign, Windows via Azure Trusted Signing, GPG tarball via +# KMS) and promoting each from the commit-sha-gated staging path to the +# canonical locations. +# +# A single LINUX step (rather than one job per platform) is feasible +# because all signing is remote-key (KMS / Trusted Signing) and every +# packaging tool is linux-capable: rcodesign signs/notarizes Apple +# artifacts cross-platform, the .dmg is built with mozilla/libdmg-hfsplus, +# the Windows installer is compiled by Inno Setup under Wine with +# Authenticode signatures from jsign, and pkgimage checksums are patched +# by a host julia. See "Publish image prerequisites" in ops/README.md. +set -euo pipefail + +# The set of arches to publish. Mirrors what the build pipeline staged. +# Each file's rows define TRIPLET (and TIMEOUT); see utilities/arches_env.sh. +ARCHES_FILES=( + .buildkite/pipelines/main/platforms/upload_linux.arches + .buildkite/pipelines/main/platforms/upload_macos.arches + .buildkite/pipelines/main/platforms/upload_windows.arches + .buildkite/pipelines/main/platforms/upload_freebsd.arches +) +# Allow callers (e.g. the scheduled no-GPL publish) to override the list. +if [[ -n "${PUBLISH_ARCHES_FILES:-}" ]]; then + # shellcheck disable=SC2206 + ARCHES_FILES=( ${PUBLISH_ARCHES_FILES} ) +fi + +# Defense in depth: refuse unless this commit is a genuine release commit on +# the canonical upstream. The real boundary is that the julia-publish +# pipeline does not build pull requests at all (see ops/README.md). +echo "--- Verify this is a trusted release commit" +bash .buildkite/utilities/verify_trusted_commit.sh + +# The trust guard above runs once; the publish role is (re-)assumed inside +# the loop below, since Buildkite OIDC tokens live at most 2h and this +# step can run longer across all triplets. +export PUBLISH_PREAUTHED=1 + +# Collect all triplets from the arches files. +TRIPLETS=() +for arches in "${ARCHES_FILES[@]}"; do + [[ -f "${arches}" ]] || { echo "WARN: missing arches file ${arches}, skipping" >&2; continue; } + while read -r env_line; do + [[ -n "${env_line}" ]] || continue + # env_line looks like: TRIPLET="x86_64-linux-gnu" TIMEOUT="30" + # shellcheck disable=SC2086 + eval "${env_line}" + [[ -n "${TRIPLET:-}" ]] && TRIPLETS+=( "${TRIPLET}" ) + done < <(bash .buildkite/utilities/arches_env.sh "${arches}") +done + +echo "--- Publishing ${#TRIPLETS[@]} triplets: ${TRIPLETS[*]}" + +# Which OIDC role to assume for signing+promotion. Defaults to the trusted +# production `publish` role; the non-production publish test stack sets +# PUBLISH_OIDC_MODE=publish-test to assume the throwaway test role instead. +PUBLISH_OIDC_MODE="${PUBLISH_OIDC_MODE:-publish}" + +FAILED=() +for triplet in "${TRIPLETS[@]}"; do + echo "+++ Publish ${triplet}" + # Fresh OIDC token per triplet (2h max lifetime; see aws_oidc.sh) + # shellcheck source=SCRIPTDIR/aws_oidc.sh + source .buildkite/utilities/aws_oidc.sh "${PUBLISH_OIDC_MODE}" + if ! TRIPLET="${triplet}" bash .buildkite/utilities/upload_julia.sh publish; then + echo "ERROR: publishing ${triplet} failed" >&2 + FAILED+=( "${triplet}" ) + fi +done + +if [[ "${#FAILED[@]}" -gt 0 ]]; then + echo "--- ${#FAILED[@]} triplet(s) failed to publish: ${FAILED[*]}" >&2 + exit 1 +fi + +echo "+++ All triplets published" diff --git a/utilities/render_launch_pipeline.py b/utilities/render_launch_pipeline.py new file mode 100644 index 000000000..2e3b572eb --- /dev/null +++ b/utilities/render_launch_pipeline.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +""" +Pre-merge renderer for the Julia Buildkite CI launch flow. + +Instead of the launch agent making ~30 separate `buildkite-agent pipeline +upload` calls (each a ~1.5s network round-trip that dominates launch latency), +this renderer produces ONE pre-grouped pipeline document on stdout. The launch +step then does a SINGLE `buildkite-agent pipeline upload` of that document. + +It reproduces, exactly, what `launch_untrusted_builders.yml` used to upload: + + * The arches-templated platform YAMLs + (pipelines/main/platforms/{build,test}_*.yml) are rendered once per arch. + The per-arch `${VAR}` / `${VAR?}` / `${VAR-default}` substitutions that the + `buildkite-agent pipeline upload` of those files used to perform are done + HERE, using the env produced by `utilities/arches_env.sh ` + plus GROUP / ALLOW_FAIL. (`USE_RR` comes from the .arches file itself where + relevant.) + + * The static / nested misc YAMLs (misc/analyzegc.yml, misc/gcext.yml, the + juliac / juliasyntax launchers, ...) are emitted VERBATIM. We do NOT + pre-resolve their variables: they contain only `$$`-runtime escapes and/or + launch-agent-env vars (e.g. `${ALLOW_FAIL?}` on gcext/test_revise). The + final single `buildkite-agent pipeline upload` of this combined document + interpolates those with the launch-agent env -- exactly as the old + per-file uploads did -- and converts `$$` -> `$`. + +CRITICAL interpolation rule: a `$$` (double dollar) is a Buildkite runtime +escape and must be PRESERVED verbatim. Per-arch substitution here only touches +single-`$` `${...}` references that are NOT preceded by another `$`. + +PowerPC: `launch_powerpc.jl` only uploads powerpc arches for Julia < 1.12. On +current master (1.14) it is a no-op, so the powerpc arches are intentionally +omitted (see OMITTED_POWERPC below). This matches the runtime behaviour. + +The result is grouped into one `group:` per label: Build, Check, Test, +Allow Fail, JuliaSyntax, JuliaC. +""" + +import os +import re +import subprocess +import sys + +# Directory of the .buildkite checkout root (this file lives in +# /utilities/render_launch_pipeline.py). +UTIL_DIR = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(UTIL_DIR) +ARCHES_ENV_SH = os.path.join(UTIL_DIR, "arches_env.sh") + +PLATFORMS = os.path.join(ROOT, "pipelines", "main", "platforms") +MISC = os.path.join(ROOT, "pipelines", "main", "misc") + + +# -------------------------------------------------------------------------- +# arches parsing (reuse arches_env.sh; do NOT reimplement .arches parsing) +# -------------------------------------------------------------------------- + +def arches_envs(arches_path): + """Return a list of dicts, one per arch row, by shelling out to + arches_env.sh and faithfully parsing each `NAME="value" ...` line via + bash `eval` so quoting matches exactly what arches_pipeline_upload.sh + would have exported.""" + out = subprocess.run( + ["bash", ARCHES_ENV_SH, arches_path], + check=True, capture_output=True, text=True, + ).stdout + + envs = [] + for line in out.splitlines(): + if not line.strip(): + continue + # The names assigned on this line (in order). + names = re.findall(r'(?:^|\s)([A-Za-z_][A-Za-z0-9_]*)=', line) + # Re-evaluate the assignment line in a clean bash so quoting is handled + # identically to `eval "export $env_map"`, then dump exactly those + # names as NUL-delimited "name=value" records to parse unambiguously + # (values may contain spaces, '=', and ','). + script = ( + 'eval "$1"; shift\n' + 'for n in "$@"; do printf "%s=%s\\0" "$n" "${!n}"; done\n' + ) + dump = subprocess.run( + ["bash", "-c", script, "_", line, *names], + check=True, capture_output=True, text=True, + ).stdout + env = {} + for rec in dump.split("\0"): + if "=" not in rec: + continue + k, v = rec.split("=", 1) + env[k] = v + envs.append(env) + return envs + + +# -------------------------------------------------------------------------- +# bash-like ${VAR} interpolation for the arches-templated YAMLs +# -------------------------------------------------------------------------- + +# Match a single-$ ${...} that is NOT preceded by another $ (i.e. not part of +# a $$ runtime escape). We assert the char before the $ is not a $. +_VAR_RE = re.compile(r'(?" +# [notify: ...] +# steps: +# - +# We want the inner step block(s) so we can re-group them. We operate on raw +# text (not a YAML round-trip) to preserve `$$` escapes and formatting exactly. +# +# These files are structurally regular, so instead of parsing them we slice the +# indented text block under a given key. The launch agents have no PyYAML, so +# the renderer must stay stdlib-only; we emit text we control (re-indented to a +# canonical depth) so indentation and `$$` stay byte-exact. + +# Canonical indentation of the emitted document: the merged inner step blocks +# live under ` steps:` of ` - group:`, so each step block's leading `- ` +# sits at 6 spaces, matching the platform sources. +STEP_INDENT = 6 + + +def _split_lines(text): + """Split into lines WITHOUT dropping a trailing newline's emptiness. + Returns the list of lines (no line endings).""" + return text.split("\n") + + +def _indent_of(line): + """Number of leading spaces; None for a blank/whitespace-only line.""" + stripped = line.lstrip(" ") + if stripped == "": + return None + return len(line) - len(stripped) + + +def _find_group_label(text, where): + """Return the group label string from the single ` - group: "