From eeb6427d2f6838e5add5fce9b7c48b88e319d041 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 12 Jul 2026 17:04:59 +0000 Subject: [PATCH] Port OIDC/KMS trust architecture to release-julia-1.13 Bring the post-#544 CI architecture to the release-1.13 branch so that release-branch PRs run under the shared `julia-pr` pipeline and release nightlies run under `julia-ci` -> `julia-publish`, instead of the old per-release pipeline slugs (`julia-release-1-dot-13`, `julia-master-scheduled`, `julia-buildkite`). Per-release differences are now expressed as branch *content* selected by julia's in-repo `.buildkite-external-version` pointer, not by runtime `pipeline.slug` checks -- so no separate Buildkite pipelines are needed (the shared `julia-ci`/`julia-publish` already branch-limit to `master release-*`). What changed: * Retire the cryptic secret-management + signed/unsigned pipeline split (cryptic_repo_keys/, secrets/, *.signature, launch_signed_jobs.yml, launch_unsigned_*.yml, launch_upload_jobs.yml, platforms/upload_*.yml). * Adopt main's OIDC + AWS KMS trust infrastructure, the single-upload render_launch_pipeline.py launcher, and the trusted julia-publish / julia-publish-test pipelines. The trailing `wait` + `trigger: julia-publish` is gated on `pipeline.slug == "julia-ci"`. Preserved as release-1.13-specific: * The build/test matrix: every pipelines/**/platforms/*.arches file is kept verbatim from release-julia-1.13 (rootfs v6.00 / docker v7.10, mmtk on the `package_linux` image, Windows CFLAGS), plus launch_powerpc.jl. * The job set: render_launch_pipeline.py is trimmed to omit the JuliaSyntax and JuliaC launch groups (new on main; not part of release-1.13 CI). The juliasyntax.* / juliac/ YAMLs remain in-tree so either can be re-enabled by re-adding its `blocks.append(...)`. NOT yet validated on Buildkite -- needs a CI run under the shared pipelines. Co-Authored-By: Claude Opus 4.8 --- .buildkite/README.md | 30 +- .buildkite/cryptic_repo_root/.buildkite | 1 - .buildkite/cryptic_repo_root/README.md | 2 - .buildkite/hooks/post-checkout | 25 +- .gitignore | 6 + Makefile | 12 - cryptic_repo_keys/.gitignore | 5 - cryptic_repo_keys/README.md | 6 - cryptic_repo_keys/repo_key.2297e5e7 | Bin 256 -> 0 bytes devdocs/macos-developer-id.md | 72 +-- devdocs/sign.md | 36 -- ops/20_export_gpg_pubkey.py | 104 ++++ ops/21_import_notary_key.sh | 81 +++ ops/22_generate_macos_csr.sh | 23 + ops/23_put_tokens.sh | 34 ++ ops/24_docs_deploy_pubkey.sh | 24 + ops/32_gen_test_codesign_cert.sh | 56 ++ ops/README.md | 329 +++++++++++ ops/common.sh | 45 ++ ops/terraform/.terraform.lock.hcl | 47 ++ ops/terraform/azure/.terraform.lock.hcl | 22 + ops/terraform/azure/main.tf | 48 ++ ops/terraform/azure/variables.tf | 10 + ops/terraform/azure/versions.tf | 13 + ops/terraform/buildkite_ids.auto.tfvars | 31 ++ ops/terraform/iam.tf | 392 +++++++++++++ ops/terraform/kms.tf | 101 ++++ ops/terraform/oidc.tf | 22 + ops/terraform/outputs.tf | 35 ++ ops/terraform/s3.tf | 73 +++ ops/terraform/test_publish.tf | 242 ++++++++ ops/terraform/variables.tf | 142 +++++ ops/terraform/versions.tf | 24 + pipelines/main/0_webui.yml | 63 ++- pipelines/main/README.md | 27 +- pipelines/main/launch_coverage.yml | 26 + pipelines/main/launch_signed_jobs.yml | 42 -- .../main/launch_signed_jobs.yml.signature | 1 - pipelines/main/launch_unsigned_builders.yml | 1 - pipelines/main/launch_unsigned_jobs.yml | 213 -------- pipelines/main/launch_untrusted_builders.yml | 69 +++ pipelines/main/launch_upload_jobs.yml | 103 ---- .../main/launch_upload_jobs.yml.signature | 1 - pipelines/main/misc/analyzegc.yml | 6 +- pipelines/main/misc/deploy_docs.yml | 78 +-- pipelines/main/misc/doctest.yml | 38 +- pipelines/main/misc/embedding.yml | 6 +- pipelines/main/misc/gcext.yml | 6 +- pipelines/main/misc/juliac/test_juliac.yml | 32 ++ .../main/misc/juliac/test_juliac_linux.arches | 9 + .../main/misc/juliac/test_juliac_linux.yml | 43 ++ .../main/misc/juliac/test_juliac_macos.arches | 8 + .../main/misc/juliac/test_juliac_macos.yml | 34 ++ .../misc/juliac/test_juliac_windows.arches | 7 + .../main/misc/juliac/test_juliac_windows.yml | 29 + pipelines/main/misc/juliasyntax.launch.yml | 31 ++ pipelines/main/misc/juliasyntax.test.yml | 215 ++++++++ pipelines/main/misc/llvmpasses.yml | 6 +- .../main/misc/pdf_docs/build_pdf_docs.yml | 2 +- pipelines/main/misc/sanitizers/asan.yml | 8 +- pipelines/main/misc/sanitizers/tsan.yml | 8 +- pipelines/main/misc/test_revise.yml | 12 +- pipelines/main/misc/trimming.yml | 6 +- .../main/misc/upload_buildkite_results.yml | 76 --- pipelines/main/misc/whitespace.yml | 6 +- pipelines/main/platforms/build_freebsd.yml | 3 + pipelines/main/platforms/build_linux.yml | 7 +- pipelines/main/platforms/build_macos.yml | 5 + pipelines/main/platforms/test_freebsd.yml | 1 + pipelines/main/platforms/test_linux.i686.yml | 3 +- pipelines/main/platforms/test_linux.yml | 3 +- pipelines/main/platforms/test_macos.yml | 1 + pipelines/main/platforms/upload_freebsd.yml | 39 -- pipelines/main/platforms/upload_linux.yml | 50 -- pipelines/main/platforms/upload_macos.yml | 42 -- pipelines/main/platforms/upload_windows.yml | 62 --- pipelines/misc/oidc_smoke_test.yml | 38 ++ pipelines/publish-test/0_webui.yml | 29 + pipelines/publish-test/launch.yml | 92 ++++ pipelines/publish/0_webui.yml | 33 ++ pipelines/publish/launch.yml | 90 +++ pipelines/scheduled/0_webui.yml | 35 +- pipelines/scheduled/coverage/coverage.yml | 114 ++-- .../scheduled/coverage/coverage.yml.signature | 1 - .../scheduled/coverage/upload_coverage.jl | 1 + pipelines/scheduled/launch_coverage.yml | 28 + pipelines/scheduled/launch_signed_jobs.yml | 47 -- .../launch_signed_jobs.yml.signature | 1 - ...jobs.yml => launch_untrusted_builders.yml} | 8 +- pipelines/scheduled/launch_upload_jobs.yml | 35 -- .../launch_upload_jobs.yml.signature | Bin 96 -> 0 bytes .../platforms/upload_linux.no_gpl.yml | 56 -- .../platforms/upload_macos.no_gpl.yml | 45 -- .../platforms/upload_windows.no_gpl.yml | 69 --- port_oidc_1.13.sh | 52 ++ secrets/.gitignore | 11 - secrets/macos_codesigning.keychain.encrypted | Bin 35216 -> 0 bytes secrets/ssh_docs_deploy.encrypted | Bin 1856 -> 0 bytes secrets/ssh_docs_deploy.pub | 1 - secrets/tarball_signing.gpg.encrypted | Bin 4832 -> 0 bytes secrets/windows_codesigning.pfx.encrypted | Bin 8832 -> 0 bytes signing-pubkeys/.gitignore | 12 + signing-pubkeys/tarball_signing.pub.asc | 28 + utilities/arches_env.sh | 10 +- utilities/aws_config | 4 - utilities/aws_oidc.sh | 142 +++++ utilities/build_envs.sh | 80 ++- utilities/build_julia.sh | 52 +- utilities/deploy_docs.sh | 14 - utilities/extract_triplet.sh | 2 + utilities/kms_gpg_sign.py | 516 ++++++++++++++++++ utilities/macos/build_dmg.sh | 176 ++++-- utilities/macos/codesign.sh | 271 ++++++--- utilities/macos/developer_id.pem | 34 ++ utilities/macos/developer_id_test.pem | 22 + utilities/macos/get_rcodesign.sh | 54 ++ utilities/macos/notary_api_key.json | 6 + utilities/macos/rcodesign/README.md | 48 ++ utilities/publish.sh | 82 +++ utilities/render_launch_pipeline.py | 493 +++++++++++++++++ utilities/rr/rr_capture.jl | 13 +- utilities/sign_tarball.sh | 18 - utilities/test_julia.sh | 63 ++- utilities/tests/oidc_smoke_test.sh | 137 +++++ utilities/tests/test_kms_gpg_sign.py | 145 +++++ utilities/update_stdlib_pkgimage_checksums.jl | 98 +++- utilities/upload_julia.sh | 365 ++++++++----- utilities/upload_to_s3.sh | 71 +++ utilities/verify_trusted_commit.sh | 86 +++ utilities/windows/check_signed.py | 60 ++ utilities/windows/codesign.sh | 191 ++++--- utilities/windows/wine_signtool.cmd | 31 ++ utilities/word.sh | 2 + 133 files changed, 6034 insertions(+), 1599 deletions(-) delete mode 120000 .buildkite/cryptic_repo_root/.buildkite delete mode 100644 .buildkite/cryptic_repo_root/README.md delete mode 100644 cryptic_repo_keys/.gitignore delete mode 100644 cryptic_repo_keys/README.md delete mode 100644 cryptic_repo_keys/repo_key.2297e5e7 delete mode 100644 devdocs/sign.md create mode 100755 ops/20_export_gpg_pubkey.py create mode 100755 ops/21_import_notary_key.sh create mode 100755 ops/22_generate_macos_csr.sh create mode 100755 ops/23_put_tokens.sh create mode 100755 ops/24_docs_deploy_pubkey.sh create mode 100755 ops/32_gen_test_codesign_cert.sh create mode 100644 ops/README.md create mode 100755 ops/common.sh create mode 100644 ops/terraform/.terraform.lock.hcl create mode 100644 ops/terraform/azure/.terraform.lock.hcl create mode 100644 ops/terraform/azure/main.tf create mode 100644 ops/terraform/azure/variables.tf create mode 100644 ops/terraform/azure/versions.tf create mode 100644 ops/terraform/buildkite_ids.auto.tfvars create mode 100644 ops/terraform/iam.tf create mode 100644 ops/terraform/kms.tf create mode 100644 ops/terraform/oidc.tf create mode 100644 ops/terraform/outputs.tf create mode 100644 ops/terraform/s3.tf create mode 100644 ops/terraform/test_publish.tf create mode 100644 ops/terraform/variables.tf create mode 100644 ops/terraform/versions.tf create mode 100644 pipelines/main/launch_coverage.yml delete mode 100644 pipelines/main/launch_signed_jobs.yml delete mode 100644 pipelines/main/launch_signed_jobs.yml.signature delete mode 120000 pipelines/main/launch_unsigned_builders.yml delete mode 100644 pipelines/main/launch_unsigned_jobs.yml create mode 100644 pipelines/main/launch_untrusted_builders.yml delete mode 100644 pipelines/main/launch_upload_jobs.yml delete mode 100644 pipelines/main/launch_upload_jobs.yml.signature create mode 100644 pipelines/main/misc/juliac/test_juliac.yml create mode 100644 pipelines/main/misc/juliac/test_juliac_linux.arches create mode 100644 pipelines/main/misc/juliac/test_juliac_linux.yml create mode 100644 pipelines/main/misc/juliac/test_juliac_macos.arches create mode 100644 pipelines/main/misc/juliac/test_juliac_macos.yml create mode 100644 pipelines/main/misc/juliac/test_juliac_windows.arches create mode 100644 pipelines/main/misc/juliac/test_juliac_windows.yml create mode 100644 pipelines/main/misc/juliasyntax.launch.yml create mode 100644 pipelines/main/misc/juliasyntax.test.yml delete mode 100644 pipelines/main/misc/upload_buildkite_results.yml delete mode 100644 pipelines/main/platforms/upload_freebsd.yml delete mode 100644 pipelines/main/platforms/upload_linux.yml delete mode 100644 pipelines/main/platforms/upload_macos.yml delete mode 100644 pipelines/main/platforms/upload_windows.yml create mode 100644 pipelines/misc/oidc_smoke_test.yml create mode 100644 pipelines/publish-test/0_webui.yml create mode 100644 pipelines/publish-test/launch.yml create mode 100644 pipelines/publish/0_webui.yml create mode 100644 pipelines/publish/launch.yml delete mode 100644 pipelines/scheduled/coverage/coverage.yml.signature create mode 100644 pipelines/scheduled/launch_coverage.yml delete mode 100644 pipelines/scheduled/launch_signed_jobs.yml delete mode 100644 pipelines/scheduled/launch_signed_jobs.yml.signature rename pipelines/scheduled/{launch_unsigned_jobs.yml => launch_untrusted_builders.yml} (91%) delete mode 100644 pipelines/scheduled/launch_upload_jobs.yml delete mode 100644 pipelines/scheduled/launch_upload_jobs.yml.signature delete mode 100644 pipelines/scheduled/platforms/upload_linux.no_gpl.yml delete mode 100644 pipelines/scheduled/platforms/upload_macos.no_gpl.yml delete mode 100644 pipelines/scheduled/platforms/upload_windows.no_gpl.yml create mode 100644 port_oidc_1.13.sh delete mode 100644 secrets/.gitignore delete mode 100644 secrets/macos_codesigning.keychain.encrypted delete mode 100644 secrets/ssh_docs_deploy.encrypted delete mode 100644 secrets/ssh_docs_deploy.pub delete mode 100644 secrets/tarball_signing.gpg.encrypted delete mode 100644 secrets/windows_codesigning.pfx.encrypted create mode 100644 signing-pubkeys/.gitignore create mode 100644 signing-pubkeys/tarball_signing.pub.asc delete mode 100644 utilities/aws_config create mode 100755 utilities/aws_oidc.sh delete mode 100644 utilities/deploy_docs.sh create mode 100755 utilities/kms_gpg_sign.py create mode 100644 utilities/macos/developer_id.pem create mode 100644 utilities/macos/developer_id_test.pem create mode 100755 utilities/macos/get_rcodesign.sh create mode 100644 utilities/macos/notary_api_key.json create mode 100644 utilities/macos/rcodesign/README.md create mode 100755 utilities/publish.sh create mode 100644 utilities/render_launch_pipeline.py delete mode 100755 utilities/sign_tarball.sh create mode 100755 utilities/tests/oidc_smoke_test.sh create mode 100755 utilities/tests/test_kms_gpg_sign.py create mode 100644 utilities/upload_to_s3.sh create mode 100755 utilities/verify_trusted_commit.sh create mode 100755 utilities/windows/check_signed.py mode change 100644 => 100755 utilities/windows/codesign.sh create mode 100644 utilities/windows/wine_signtool.cmd diff --git a/.buildkite/README.md b/.buildkite/README.md index e22aebcc7..3f5f89c08 100644 --- a/.buildkite/README.md +++ b/.buildkite/README.md @@ -1,3 +1,31 @@ # This directory not used by base Julia! -This directory is for performing CI on the `julia-buildkite` repository itself! It does not get used at all during the actual build for Julia itself. +This directory is for performing CI on the `julia-buildkite` repository +itself! It does not get used at all during the actual build for Julia +itself. + +## How the self-test works + +The `julia-buildkite-ci` Buildkite pipeline (untrusted; same cluster as +`julia-pr` / `julia-ci`) builds this repository's pull requests and +`main`. Its webUI configuration is the same "Launch pipelines" step as +the julia build pipelines (see `pipelines/main/0_webui.yml`), but the +pipeline's *repository* is `JuliaCI/julia-buildkite` -- so every job of +the build checks out this repository and runs `hooks/post-checkout`, +which: + +1. replaces the working directory with a `JuliaLang/julia` checkout + (`UPSTREAM_BRANCH`, default `master`; pinned once per build via + meta-data so all jobs test the same julia commit), and +2. pins the external-buildkite plugin's build meta-data to + `${BUILDKITE_COMMIT}` (this build's julia-buildkite commit), so every + launched job runs the proposed pipeline code. + +Trust model (see `ops/README.md`): the self-test pipeline holds no +secrets, tokens, or signing rights. Build jobs stage their artifacts +write-once to the pipeline's own ephemeral bucket +(`julialang-ephemeral-buildkite`) via its own OIDC role +(`julia-oidc-stage-buildkite`) -- separate from the buckets juliaup and +julia-publish consume, so a self-test build can never place anything a +production consumer would read. The julia-publish trigger is gated on +`pipeline.slug == "julia-ci"` and so never fires here. diff --git a/.buildkite/cryptic_repo_root/.buildkite b/.buildkite/cryptic_repo_root/.buildkite deleted file mode 120000 index c25bddb6d..000000000 --- a/.buildkite/cryptic_repo_root/.buildkite +++ /dev/null @@ -1 +0,0 @@ -../.. \ No newline at end of file diff --git a/.buildkite/cryptic_repo_root/README.md b/.buildkite/cryptic_repo_root/README.md deleted file mode 100644 index c0c323dd7..000000000 --- a/.buildkite/cryptic_repo_root/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This directory is used to fake out `cryptic`'s search algorithms. -Execute things like `verify_treehashes --repo-root=$(pwd)` from within this directory. diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index be42e58ad..985974ca1 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,18 +1,26 @@ #!/usr/bin/env bash +# Repository hook for the julia-buildkite SELF-TEST pipeline: its jobs +# check out this repository, and this hook then (1) swaps the working +# directory for a JuliaLang/julia checkout (pinned once per build via +# meta-data, so every job tests the same julia commit) and (2) points the +# external-buildkite plugin's meta-data at this build's commit -- so the +# whole build/test flow runs against the proposed julia-buildkite code. +# See .buildkite/README.md. + set -euo pipefail # Buildkite users can trigger new builds with these values overridden to # test specific julia gitshas UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/JuliaLang/julia.git}" -UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-release-1.13}" +UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-master}" -# All of our workers are required to provide a default for the julia cache dir -# We're going to cache repositories in here, just like buildkite itself would: -UPSTREAM_CACHE="${BUILDKITE_PLUGIN_JULIA_CACHE_DIR?}/repos/$(tr ':/.' '---' <<<"${UPSTREAM_URL}")" +# Cache repositories in the worker's julia cache dir (if it provides one), +# just like buildkite itself would: +UPSTREAM_CACHE="${BUILDKITE_PLUGIN_JULIA_CACHE_DIR:-}/repos/$(tr ':/.' '---' <<<"${UPSTREAM_URL}")" DISABLE_UPSTREAM_CACHE="false" -if [[ "$(uname)" == "FreeBSD" ]]; then +if [[ "$(uname)" == "FreeBSD" || -z "${BUILDKITE_PLUGIN_JULIA_CACHE_DIR:-}" ]]; then DISABLE_UPSTREAM_CACHE="true" fi @@ -96,6 +104,7 @@ if ! buildkite-agent meta-data exists BUILDKITE_JULIA_VERSION; then buildkite-agent meta-data set BUILDKITE_JULIA_BRANCH "${UPSTREAM_BRANCH}" fi -# Export S3_BUCKET and S3_BUCKET_PREFIX, to force our CI to upload to a different S3 target than the actual CI pipeline would. -export S3_BUCKET="julialang-ephemeral" -export S3_BUCKET_PREFIX="julia-buildkite-uploads/bin" +# No S3 redirection is needed here: staging-bucket selection is keyed on +# the pipeline slug (utilities/build_envs.sh), so self-test jobs stage to +# the pipeline's own ephemeral bucket, and IAM only grants this pipeline's +# OIDC identity that bucket anyway (see ops/terraform/iam.tf). diff --git a/.gitignore b/.gitignore index 7a9708ad2..e58d87649 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ cryptic-buildkite-plugin/ # Ignore IDE files .vscode/ +__pycache__/ + +# Terraform working directories (provider binaries); lock files are committed +.terraform/ +*.tfstate +*.tfstate.* diff --git a/Makefile b/Makefile index 79476060e..3e5e9eb93 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,2 @@ -export PATH := $(shell pwd)/cryptic-buildkite-plugin/bin:$(PATH) - -decrypt: - cd .buildkite/cryptic_repo_root && decrypt --repo-root=$$(pwd) --verbose - -sign_treehashes: - cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose - git commit -a -m "sign treehashes" - -verify_treehashes: - cd .buildkite/cryptic_repo_root && verify_treehashes --repo-root=$$(pwd) --verbose - print-%: echo "$*=$($*)" diff --git a/cryptic_repo_keys/.gitignore b/cryptic_repo_keys/.gitignore deleted file mode 100644 index 8d18931db..000000000 --- a/cryptic_repo_keys/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore the unencrypted repo_key -repo_key - -# Ignore any agent keys (public or private) we have stored -agent_key* diff --git a/cryptic_repo_keys/README.md b/cryptic_repo_keys/README.md deleted file mode 100644 index a66b6f926..000000000 --- a/cryptic_repo_keys/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## Cryptic repository keys - -This folder contains RSA-encrypted symmetric AES keys. -These are used by buildkite agents to decrypt the secrets embedded within this repository. -Each buildkite agent contains an RSA secret key that is used to unlock the symmetric AES key that was used to encrypt the secrets within this repository. -For more information, see the [`cryptic` buildkite plugin repository](https://github.com/JuliaCI/cryptic-buildkite-plugin). diff --git a/cryptic_repo_keys/repo_key.2297e5e7 b/cryptic_repo_keys/repo_key.2297e5e7 deleted file mode 100644 index 2ab9198b4ce2d7c7a9327935e18aaef5943629d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 256 zcmV+b0ssE4{g>pIb#Eq&=vC$Qr^8S_O6!cOYE`=}q%}#an@eaJd$RWGY{B`_JI5h4 zkH`Ok?yS%>4AD6gqiiL&E|`(uBGp1*D8z1}bTOSJGyXKmUCrMUN%Q;gJ{M_gVt6DF z#aRD2;iY!|^9jgEcBXhT=jxw2yplIkGkb~?zh)kpuL|wT!R7{E{^jQhsqTt2Woi5X z6&zlfSSPoyKVU`TIJAL8I6(F*9}>v<(M$c2U_*AHb;CSzxbLIaZK0;0gYIZ?hzwB# zq^tZ&SA{ud8h!8dRd!Rzp7pWdE3F(RM#CkMHZoXOfk*SkHmzrDCiOUb@l`xQjdi^f GB)!2VDuP}B diff --git a/devdocs/macos-developer-id.md b/devdocs/macos-developer-id.md index 281b992b2..93bfaa162 100644 --- a/devdocs/macos-developer-id.md +++ b/devdocs/macos-developer-id.md @@ -1,54 +1,40 @@ -## Expired certificates - -The main symptom will be that the upload job will be failing. The unlock -keychain step prints the status of the certificate and it will print something -like (`CSSMERR_TP_CERT_EXPIRED`) if the certificate is expired. - -### Update expired Apple Developer ID - -To replace the certificate you will need first a MacOS machine and to emit a new -certificate. - -1. To replace an expired Apple certificate, clone `julia-buildkite` repo and - clone https://github.com/JuliaCI/cryptic-buildkite-plugin in its root. - You will also need to have the private `agent.key`, and point towards it - by defining the `AGENT_PRIVATE_KEY_PATH` environment variable. -2. You can decrypt by running `make decrypt`. -3. Get the `macos_codesigning.keychain` file in the `secrets` directory and add - it to your local keychains with the Keychain Access app (File > Import - Items). -4. From that app delete the old certificate and add the new one (it's a `.cer` - file). -5. Test the certificate by right clicking and running both the general - evaluation and the codesigning one. -6. Update the identity in `MACOS_CODESIGN_IDENTITY` (You can find the identity - by doing `security find-identity -p codesigning $(PATH_TO_KEYCHAIN)/macos_codesigning.keychain` ). -7. You can also test it by running the codesign.sh script in this repo with - `./utilities/macos/codesign.sh --keychain ./secrets/macos_codesigning.keychain --identity $(NEW_IDENTITY) ./test` - with some executable. -8. Afterward reencrypt the keychain by running - `./cryptic-buildkite-plugin/bin/encrypt_file --private-key=$(INSERT_PRIVATE_KEY) --repo-key=$(INSERT_REPO_KEY) ./secrets/macos_codesigning.keychain` -9. Finally sign the repo with `make sign_treehashes` - -The `security` cli app is also useful for debugging and managing keychains. You -can find more information about it by running `man security` in the terminal. +# macOS Developer ID certificate +The Developer ID Application private key lives in AWS KMS +(`alias/julia-macos-codesigning`); macOS codesigning happens via +`rcodesign` with the AWS KMS backend (see `utilities/macos/rcodesign/`). +The certificate itself is public and committed at +`utilities/macos/developer_id.pem`. + +## Expired certificate + +The main symptom will be failing upload jobs (signature validation errors +or Apple rejecting the notarization). To renew: + +1. Generate a fresh CSR from the KMS key (no new key needed): + `ops/22_generate_macos_csr.sh` +2. Submit it at https://developer.apple.com/account/resources/certificates + (type: "Developer ID Application"), download the `.cer`. +3. Convert and commit: + `openssl x509 -inform DER -in developerID_application.cer -out utilities/macos/developer_id.pem` +4. Test locally (requires kms:Sign on the key): + `rcodesign sign --aws-kms-key alias/julia-macos-codesigning --aws-kms-certificate-file utilities/macos/developer_id.pem ` ## New agreements -It is also possible that just a new agreement is needed. In that case, you will -see the following error message returned by the `codesign` command: +It is also possible that just a new agreement is needed. In that case, you +will see the following error message during notarization: -> HTTP status code: 403. A required agreement is missing or has expired. This -> request requires an in-effect agreement that has not been signed or has -> expired. Ensure your team has signed the necessary legal agreements and that -> they are not expired. +> HTTP status code: 403. A required agreement is missing or has expired. +> This request requires an in-effect agreement that has not been signed or +> has expired. Ensure your team has signed the necessary legal agreements +> and that they are not expired. -In this case, it is sufficient to visit `developer.apple.com` and logging in +In this case, it is sufficient to visit `developer.apple.com` and log in using the Apple ID that is associated with the Apple Developer account. You will be prompted to accept the new agreement. After that, the build should succeed. -Note that currently, the Apple ID that's used to sign Julia binaries is owned -by JuliaHub, so you will need to get in touch with somebody from the +Note that currently, the Apple ID that's used to sign Julia binaries is +owned by JuliaHub, so you will need to get in touch with somebody from the organization to accept the agreement. diff --git a/devdocs/sign.md b/devdocs/sign.md deleted file mode 100644 index 19358c006..000000000 --- a/devdocs/sign.md +++ /dev/null @@ -1,36 +0,0 @@ -## Dependencies - -1. [shyaml](https://github.com/0k/shyaml) - -## Instructions - -If you are a maintainer, and you want to re-sign all of the signatures, here are the steps: - -### If you only have access to the _repository_ private key - -```shell -git clone git@github.com:JuliaCI/julia-buildkite.git -cd julia-buildkite -git checkout YOURINITIALS/YOUR-BRANCH-NAME -git clone https://github.com/JuliaCI/cryptic-buildkite-plugin.git -mv /path/to/my/repository/private/key ./cryptic_repo_keys/repo_key -make sign_treehashes -git push origin YOURINITIALS/YOUR-BRANCH-NAME -``` - -### If you have access to the _agent_ private key - -``` -git clone git@github.com:JuliaCI/julia-buildkite.git -cd julia-buildkite -git checkout YOURINITIALS/YOUR-BRANCH-NAME -git clone https://github.com/JuliaCI/cryptic-buildkite-plugin.git -cd cryptic-buildkite-plugin -git checkout main -cd .. -export AGENT_PRIVATE_KEY_PATH=/path/to/my/agent.key -make sign_treehashes -unset AGENT_PRIVATE_KEY_PATH -rm -f /path/to/my/agent.key -git push origin YOURINITIALS/YOUR-BRANCH-NAME -``` diff --git a/ops/20_export_gpg_pubkey.py b/ops/20_export_gpg_pubkey.py new file mode 100755 index 000000000..393132d4c --- /dev/null +++ b/ops/20_export_gpg_pubkey.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# This file is a part of Julia. License is MIT: https://julialang.org/license +"""Export the KMS-held release tarball signing key as an OpenPGP public key. + +The signing key is generated inside KMS (alias/julia-tarball-signing, +created by ops/terraform) and never leaves it. This script fetches the +RSA public half via `aws kms get-public-key`, wraps it in an OpenPGP v4 +public key packet + user ID, and obtains the positive self-certification +with a `kms:Sign` call -- producing an armored public key block that GPG +and friends accept. Commit the output (signing-pubkeys/tarball_signing.pub.asc) +and publish it as the Julia releases signing key; release signatures +made by utilities/kms_gpg_sign.py verify against it. + +The OpenPGP fingerprint covers the key creation timestamp. It defaults to +the KMS key's own CreationDate (kms:DescribeKey) -- an immutable, already- +pinned value that also matches what kms_gpg_sign.py's --public-key-from-kms +signing path derives -- so re-running reproduces the identical key block. +--created overrides it only to reproduce a key minted under a different +timestamp; a changed timestamp would mint a "new" identity for the same key. + +Usage: + 20_export_gpg_pubkey.py [--created YYYY-MM-DD] [--kms-key-id ALIAS] [-o FILE] + +For testing without KMS access, `--local-key key.pem` uses a local RSA +private key via openssl, exercising the identical construction path. + +Requires AWS credentials with kms:GetPublicKey + kms:Sign on the key. +""" + +import argparse +import os +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(SCRIPT_DIR, "..", "utilities")) + +import kms_gpg_sign as K # noqa: E402 + +# The DER/SPKI parsing, kms:GetPublicKey fetch, and --created parsing live in +# kms_gpg_sign.py (shared with its --public-key-from-kms signing mode). + +DEFAULT_KMS_KEY = "alias/julia-tarball-signing" +DEFAULT_UID = "Julia Release Signing Key " +DEFAULT_OUTPUT = os.path.join(SCRIPT_DIR, "..", "signing-pubkeys", "tarball_signing.pub.asc") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--created", + help="override the key creation time (unix timestamp or " + "YYYY-MM-DD UTC midnight); part of the key fingerprint. " + "Defaults to the KMS key's own CreationDate") + parser.add_argument("--uid", default=DEFAULT_UID, + help=f"user ID for the key (default: {DEFAULT_UID!r})") + parser.add_argument("-o", "--output", default=DEFAULT_OUTPUT, + help="output path for the armored public key block") + group = parser.add_mutually_exclusive_group() + group.add_argument("--kms-key-id", default=DEFAULT_KMS_KEY, + help=f"AWS KMS key ID/ARN/alias (default: {DEFAULT_KMS_KEY})") + group.add_argument("--local-key", + help="local RSA private key PEM (testing only)") + args = parser.parse_args() + + if args.created: + timestamp = K.parse_created(args.created) + elif args.local_key: + # A local test key has no KMS CreationDate; pin a fixed epoch so the + # construction path stays deterministic. + timestamp = 0 + else: + # Pin the fingerprint to the KMS key's immutable provisioning date, + # matching kms_gpg_sign.py's --public-key-from-kms signing path. + timestamp = K.kms_key_creation_date(args.kms_key_id) + + if args.local_key: + spki = K.local_public_key(args.local_key) + signer = K.LocalOpensslSigner(args.local_key) + else: + spki = K.kms_public_key(args.kms_key_id) + signer = K.KmsSigner(args.kms_key_id) + + n, e = K.rsa_components_from_spki(spki) + key_body = K.build_rsa_key_body(n, e, timestamp) + pubkey, cert = K.build_certificate(signer, key_body, args.uid, timestamp) + armored = K.armor(cert, "PUBLIC KEY BLOCK") + + with open(args.output, "w") as f: + f.write(armored) + + # Round-trip self-check: what we wrote parses back to the same key. + assert K.load_public_key(args.output).fingerprint == pubkey.fingerprint + + print(f"Fingerprint: {pubkey.fingerprint.hex().upper()}") + print(f"User ID: {args.uid}") + print(f"Created: {timestamp}") + print(f"Written to: {args.output}") + print() + print("Commit this file, and publish it as the Julia releases signing key") + print("(it replaces the pre-migration juliareleases.asc).") + + +if __name__ == "__main__": + main() diff --git a/ops/21_import_notary_key.sh b/ops/21_import_notary_key.sh new file mode 100755 index 000000000..edce99b72 --- /dev/null +++ b/ops/21_import_notary_key.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Import the App Store Connect API private key (.p8, ECDSA P-256) into the +# alias/julia-notary-api KMS key (EXTERNAL origin), then emit the unified +# api key JSON used by rcodesign (contains no secret material). +# +# Usage: 21_import_notary_key.sh AuthKey_XXXXXXXXXX.p8 +# +# Apple generates ASC API keys, so the existing key must be imported (we +# cannot register our own public key with Apple). Run from a trusted +# machine; securely delete the .p8 afterwards. +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +P8_FILE="${1:?usage: $0 AuthKey.p8 }" +ISSUER_ID="${2:?usage: $0 AuthKey.p8 }" +API_KEY_ID="${3:?usage: $0 AuthKey.p8 }" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# GetParametersForImport / ImportKeyMaterial reject aliases; resolve to the ARN. +KEY_ARN="$(kms_key_arn "${KMS_ALIAS_NOTARY_API}")" +if [ -z "${KEY_ARN}" ]; then + echo "ERROR: ${KMS_ALIAS_NOTARY_API} not found (run terraform apply first)" >&2 + exit 1 +fi + +echo "--- Convert .p8 (PEM PKCS#8) to DER" +openssl pkcs8 -topk8 -nocrypt -in "${P8_FILE}" -outform DER -out "${WORK}/key.pkcs8.der" + +echo "--- Get KMS import parameters" +aws kms get-parameters-for-import --region "${AWS_REGION}" \ + --key-id "${KEY_ARN}" \ + --wrapping-algorithm RSA_AES_KEY_WRAP_SHA_256 \ + --wrapping-key-spec RSA_4096 \ + --output json > "${WORK}/import-params.json" + +python3 - "${WORK}" <<'EOF' +import base64, json, sys +work = sys.argv[1] +params = json.load(open(f"{work}/import-params.json")) +open(f"{work}/wrapping_key.der", "wb").write(base64.b64decode(params["PublicKey"])) +open(f"{work}/import_token.bin", "wb").write(base64.b64decode(params["ImportToken"])) +EOF + +echo "--- Wrap key material" +openssl rand 32 > "${WORK}/aes.key" +openssl pkeyutl -encrypt \ + -pubin -inkey "${WORK}/wrapping_key.der" -keyform DER \ + -pkeyopt rsa_padding_mode:oaep \ + -pkeyopt rsa_oaep_md:sha256 \ + -pkeyopt rsa_mgf1_md:sha256 \ + -in "${WORK}/aes.key" -out "${WORK}/wrapped_aes.bin" +openssl enc -id-aes256-wrap-pad \ + -K "$(xxd -p "${WORK}/aes.key" | tr -d '\n')" \ + -iv A65959A6 \ + -in "${WORK}/key.pkcs8.der" -out "${WORK}/wrapped_key.bin" +cat "${WORK}/wrapped_aes.bin" "${WORK}/wrapped_key.bin" > "${WORK}/encrypted_material.bin" + +echo "--- Import key material into ${KMS_ALIAS_NOTARY_API}" +aws kms import-key-material --region "${AWS_REGION}" \ + --key-id "${KEY_ARN}" \ + --encrypted-key-material "fileb://${WORK}/encrypted_material.bin" \ + --import-token "fileb://${WORK}/import_token.bin" \ + --expiration-model KEY_MATERIAL_DOES_NOT_EXPIRE + +OUT_JSON="${SCRIPT_DIR}/../utilities/macos/notary_api_key.json" + +echo "--- Write unified api key JSON (no secret material) to ${OUT_JSON}" +cat > "${OUT_JSON}" < /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +RCODESIGN="${RCODESIGN:-rcodesign}" + +"${RCODESIGN}" generate-certificate-signing-request \ + --aws-kms-key "${KMS_ALIAS_MACOS_CODESIGN}" \ + --aws-kms-region "${AWS_REGION}" \ + --csr-pem-file julia-macos-codesigning.csr.pem + +echo "CSR written to julia-macos-codesigning.csr.pem" diff --git a/ops/23_put_tokens.sh b/ops/23_put_tokens.sh new file mode 100755 index 000000000..7908cb562 --- /dev/null +++ b/ops/23_put_tokens.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Store a CI telemetry bearer token in SSM Parameter Store. +# +# Bearer tokens (codecov, coveralls, buildkite analytics) are inherently +# symmetric secrets and cannot be turned into KMS signing operations, so +# they live in the AWS secrets store and are fetched at job runtime by +# the julia-oidc-tokens-ci role (julia-ci only; PR builds get no +# tokens). Nothing secret is stored in the repository. +# +# Usage: +# 23_put_tokens.sh codecov_token # prompts for the value +# 23_put_tokens.sh coveralls_token +# 23_put_tokens.sh buildkite_analytics_token +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +NAME="${1:?usage: $0 }" + +if [[ ! "${NAME}" =~ ^[a-z0-9_]+$ ]]; then + echo "ERROR: token name must be lowercase [a-z0-9_]" >&2 + exit 1 +fi + +read -r -s -p "Value for ${SSM_TOKEN_PREFIX}/${NAME}: " VALUE +echo + +aws ssm put-parameter --region "${AWS_REGION}" \ + --name "${SSM_TOKEN_PREFIX}/${NAME}" \ + --type SecureString \ + --value "${VALUE}" \ + --overwrite + +echo "Stored ${SSM_TOKEN_PREFIX}/${NAME}" diff --git a/ops/24_docs_deploy_pubkey.sh b/ops/24_docs_deploy_pubkey.sh new file mode 100755 index 000000000..ad718992f --- /dev/null +++ b/ops/24_docs_deploy_pubkey.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Print the SSH public key of the KMS-held docs deploy key, in OpenSSH +# format. Register this as a deploy key (with write access) on +# github.com/JuliaLang/docs.julialang.org. +# +# The private half never leaves KMS: the docs deploy job's ssh uses the +# aws-kms-pkcs11 provider (https://github.com/JackOfMostTrades/aws-kms-pkcs11) +# so each SSH authentication is a kms:Sign call gated by the +# julia-oidc-docs-deploy role. +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +aws kms get-public-key --region "${AWS_REGION}" \ + --key-id "${KMS_ALIAS_DOCS_DEPLOY}" \ + --query PublicKey --output text | base64 -d > "${WORK}/spki.der" + +openssl pkey -pubin -inform DER -in "${WORK}/spki.der" -out "${WORK}/pub.pem" +ssh-keygen -i -m PKCS8 -f "${WORK}/pub.pem" > "${WORK}/id.pub" + +echo "$(cat "${WORK}/id.pub") julia-docs-deploy@kms" diff --git a/ops/32_gen_test_codesign_cert.sh b/ops/32_gen_test_codesign_cert.sh new file mode 100755 index 000000000..e9e483308 --- /dev/null +++ b/ops/32_gen_test_codesign_cert.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Generate a SELF-ISSUED (throwaway) macOS codesigning certificate for the +# NON-PRODUCTION test KMS key alias/julia-macos-codesigning-test, and commit it +# as utilities/macos/developer_id_test.pem. +# +# This is the test-stack counterpart of ops/22_generate_macos_csr.sh. Unlike +# production -- where the CSR is submitted to Apple and the issued Developer ID +# certificate is committed as developer_id.pem -- here we never involve Apple: +# we mint a CSR from the test KMS key (which carries the key's PUBLIC half, +# self-signed by KMS to prove possession), then issue a leaf certificate for it +# from a throwaway local CA. The resulting cert therefore contains the test KMS +# key's public key, so rcodesign signatures made with that KMS key verify +# against it. It does NOT chain to Apple and Gatekeeper would reject it -- but +# the test stack only exercises the SIGNING mechanics (rcodesign -> kms:Sign), +# not Gatekeeper acceptance, and macOS notarization is skipped entirely +# (PUBLISH_SKIP_NOTARIZATION=1). +# +# One-time: run with AWS credentials that have kms:GetPublicKey + kms:Sign on +# the test key (e.g. after `terraform apply` with buildkite_test_pipeline_id +# set), then commit the output. Requires the patched rcodesign and openssl. +set -euo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/common.sh" + +RCODESIGN="${RCODESIGN:-rcodesign}" +OUT="${SCRIPT_DIR}/../utilities/macos/developer_id_test.pem" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# 1. CSR from the test KMS key (carries its public key; signed by KMS). +"${RCODESIGN}" generate-certificate-signing-request \ + --aws-kms-key "${KMS_ALIAS_MACOS_CODESIGN_TEST}" \ + --aws-kms-region "${AWS_REGION}" \ + --csr-pem-file "${WORK}/kms.csr" + +# 2. Throwaway local CA (its private key is discarded with $WORK). +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "${WORK}/ca.key" -out "${WORK}/ca.pem" -days 3650 -sha256 \ + -subj "/CN=Julia TEST codesigning CA (NOT FOR PRODUCTION)/O=The Julia Project (TEST)/C=US" + +# 3. Issue the leaf cert for the KMS public key, with the codesigning EKU. +cat > "${WORK}/leaf.ext" <<'EOF' +keyUsage = critical, digitalSignature +extendedKeyUsage = critical, codeSigning +basicConstraints = critical, CA:FALSE +EOF +openssl x509 -req -in "${WORK}/kms.csr" \ + -CA "${WORK}/ca.pem" -CAkey "${WORK}/ca.key" -CAcreateserial \ + -days 3650 -sha256 -extfile "${WORK}/leaf.ext" \ + -out "${OUT}" + +echo "Wrote self-issued test codesigning certificate to ${OUT}" +openssl x509 -in "${OUT}" -noout -subject -issuer -dates +echo +echo "Commit ${OUT} (a public, non-secret, non-production certificate)." diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 000000000..65963fa90 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,329 @@ +# Julia CI trust infrastructure (post-cryptic) + +This directory configures the AWS (and Azure) resources that replace the +`cryptic` Buildkite plugin: + +- **`terraform/`** — the declarative infrastructure: Buildkite OIDC + provider, the four KMS keys, the IAM roles and their policies. +- **`terraform/azure/`** — a separate root module (different control + plane / credentials) for the Azure Trusted Signing federated identity + credentials. +- **numbered scripts** — the imperative one-time operations: importing + existing key material into KMS, entering bearer tokens, generating the + macOS CSR, printing the docs deploy public key, uploading tool binaries. + These stay out of Terraform on purpose: their inputs are secrets, and + anything Terraform touches ends up in its state file. The Terraform + state for this module therefore contains **no secret material** and can + be stored in any ordinary backend. After this migration, **no private key ever +exists on a build agent or in this repository** — every signature is a +remote KMS operation, authorized by the job's OIDC identity: + +| Concern | Before (cryptic) | After | +|--------------------------|-------------------------------------------|--------------------------------------------------------------| +| S3 uploads | static `AWS_ACCESS_KEY_ID/SECRET` in yml | OIDC → `julia-oidc-stage-{pr,ci}` (untrusted, per pipeline) + `julia-oidc-publish` (trusted) roles | +| macOS codesigning | keychain file w/ Developer ID key | KMS RSA key + patched `rcodesign` (`utilities/macos/rcodesign`) | +| macOS notarization | Apple ID + app-specific password | App Store Connect API key in KMS (ES256 JWTs via `kms:Sign`) | +| Linux/source GPG signing | raw GPG private key file | fresh key generated in KMS, `utilities/kms_gpg_sign.py` (new public key published) | +| Windows codesigning | `AZURE_CLIENT_SECRET` | Azure workload identity federation (Buildkite OIDC) | +| Docs deploy SSH key | cryptic-encrypted key file | SSH key in KMS, ssh signs via [aws-kms-pkcs11](https://github.com/JackOfMostTrades/aws-kms-pkcs11) | +| Telemetry bearer tokens | cryptic-encrypted variables | SSM Parameter Store (SecureString), OIDC-gated `ssm:GetParameter` | + +(The codecov / coveralls / buildkite-analytics tokens are bearer tokens — +there is no public-key operation to delegate — so they live in the AWS +secrets store and are fetched at runtime; they are never stored in the +repository in any form.) + +## Trusted / untrusted pipeline split + +The single most important control is that **pull requests and release +publishing run in different Buildkite pipelines**, and the trusted IAM roles +only trust the publish pipeline's slug. There are three pipelines: + +- **`julia-pr`** — builds pull requests (untrusted). +- **`julia-ci`** — builds trusted refs only: master, release-*, tags, and the + scheduled nightlies (untrusted to sign, but it is what triggers publish). +- **`julia-publish`** — the trusted pipeline that signs + promotes. + +A fourth, untrusted pipeline, **`julia-buildkite-ci`**, is the +julia-buildkite repository's own self-test CI (see `.buildkite/README.md`): +it builds julia with *proposed* pipeline code, staging write-once to its own +bucket (`julialang-ephemeral-buildkite`, role `julia-oidc-stage-buildkite`) +that neither juliaup nor julia-publish ever reads. It gets no tokens and no +publish trigger, exactly like `julia-pr`. + +``` + julia-pr (pull requests) julia-ci (master / release-* / tags / scheduled) + build ──► s3://julialang-ephemeral-pr/ build ──► s3://julialang-ephemeral-ci/ + //julia-* //julia-* + (UNTRUSTED: the build step stages directly -- write-once, own pipeline's + ephemeral bucket, own commit's path; per-pipeline roles + julia-oidc-stage-pr / julia-oidc-stage-ci, no KMS) + PRs stop here (juliaup reads │ tests pass, julia-ci only: + the -pr bucket). ▼ trigger + julia-publish ──► publish_all (single step) + (TRUSTED: role julia-oidc-publish, kms:Sign + read julia-ci staging bucket + ONLY + write final) + verify_trusted_commit.sh → sign (rcodesign / Trusted Signing / KMS-GPG) → promote → deploy docs +``` + +Why this is safe where branch-pinning was not: Buildkite reports a pull +request build's `sub` ref as the PR head branch, with no PR-vs-push +discriminator, so a fork PR whose branch is named `master` would match a +`ref:refs/heads/master` trust pattern. We therefore do **not** trust any +build-pipeline slug for signing. The trusted roles trust only the +`julia-publish` slug, and a PR (which only ever runs in `julia-pr`) cannot +produce a build under that slug. + +**Required Buildkite configuration** (this is load-bearing — the IAM trust +depends on it): +- `julia-pr`: builds pull requests (this is the only pipeline that should). +- `julia-ci`: builds master / release-* / tags / schedule; **does not build + pull requests**. +- `julia-publish`: "Build pull requests" OFF (incl. third-party forks); + branch-limited to `master release-*` (plus build tags `v*`); triggered only + by `julia-ci`. Its webUI step loads launch steps from a pinned + julia-buildkite (the external-buildkite plugin), never from the triggered + build's tree. +- Backstop: every publish job runs `utilities/verify_trusted_commit.sh`, + which aborts unless `BUILDKITE_COMMIT` is reachable from a protected ref + of the canonical upstream — so even a mis-triggered build cannot publish. + +The publish step runs on a single linux `queue: publish` agent: every +signature is remote-key (KMS / Trusted Signing) and all packaging tooling +is ported to linux (see "Publish image prerequisites" below). The macOS +Julia.app is assembled by the build_ step (`make -C contrib/mac/app`, which +already runs on a Mac so `osacompile` is available) and staged unsigned; the +publish step only codesigns it and wraps it in the `.dmg`, so no Mac is +needed there. + +## Trust model + +Buildkite agents mint OIDC tokens (`buildkite-agent oidc request-token`) +whose `sub` claim is `organization::pipeline::ref::commit::step:` +and which carry AWS session tags (`step_key`, `build_commit`, `pipeline_slug`, ...). + +* **Trusted roles** (`julia-oidc-publish`, `julia-oidc-docs-deploy`) are + assumable only from the `julia-publish` pipeline slug, and only from the + expected step (`aws:RequestTag/step_key` in the trust policy). +* Every trust policy additionally pins the Buildkite **organization, + pipeline, and cluster UUIDs** (`aws:RequestTag/organization_id` / + `pipeline_id` / `cluster_id`, values in + `ops/terraform/buildkite_ids.auto.tfvars`). Slugs are renameable and can + be re-minted by deleting + recreating a pipeline; the UUIDs cannot, so a + recreated pipeline with a matching slug does not regain role access. + IAM can only condition on `aud`/`sub` from the raw OIDC token, which is + why these (like `step_key`) travel as AWS session tags requested in + `utilities/aws_oidc.sh`; the claims are attested by Buildkite, and a + token requested without the tags fails the trust conditions outright. +* **Staging**: each build pipeline has its own role + (`julia-oidc-stage-pr` / `julia-oidc-stage-ci`), assumable from any job + of that one pipeline (the build step assumes it directly; there is no + step restriction on the untrusted roles). Each role can *only* write to + its own pipeline's staging bucket, and only below + `/${aws:PrincipalTag/build_commit}/*` — the source git sha, + an attested session tag the job cannot influence. The buckets + (`julialang-ephemeral-pr` / `julialang-ephemeral-ci`) are ephemeral and + lifecycle-expired, and publish reads **only** the `julia-ci` one — so a + PR build can never place, or pre-claim (paths are write-once), anything + that publish would consume. Build / PR jobs cannot touch release paths + or sign anything. (Consumers, e.g. juliaup, map PR number → head sha + via the GitHub API and fetch from the sha path in the `-pr` bucket.) +* **Tokens**: only `julia-ci` has a tokens role (`julia-oidc-tokens-ci`, + SSM `ssm:GetParameter` on the telemetry tokens). There is deliberately + no `-pr` counterpart: a pull request executes attacker-controlled code + inside the job, which could exfiltrate any bearer token the job can + read — so PR builds hold **no tokens at all** (and consequently no + coverage/analytics uploads happen on PRs). +* **No overwrites**: all roles must use S3 conditional writes + (`If-None-Match: *`, enforced via the `s3:if-none-match` policy condition); + uploads of already-existing objects fail. The only exception is the + `julia-latest-*` pointer objects, which release builds intentionally + repoint. Object versioning on the bucket is recommended belt-and-braces. +* Signing never exposes key material: every signature (macOS code signature, + notarization JWT, GPG tarball signature, docs-deploy SSH authentication) + is a `kms:Sign` call, conditioned on the calling job's step + (`aws:PrincipalTag/step_key`). + +## Runbook + +One-time setup, in order (admin AWS credentials; region and bucket names +are Terraform variables with the production defaults): + +1. Create the four Buildkite pipelines and set their WebUI steps: + - `julia-pr` — builds pull requests; WebUI = `pipelines/main/0_webui.yml`. + - `julia-ci` — builds master / release-* / tags / schedule (no PRs); + WebUI = `pipelines/main/0_webui.yml` (same launch flow; the rendered + publish trigger is gated on `pipeline.slug == "julia-ci"`, see + `utilities/render_launch_pipeline.py`). + - `julia-publish` — PRs OFF, branch-limited, triggered by `julia-ci`; + WebUI = `pipelines/publish/0_webui.yml`. + - `julia-buildkite-ci` — repository `JuliaCI/julia-buildkite`, PRs ON + (incl. forks), branch builds limited to `main`; WebUI = + `pipelines/main/0_webui.yml` again (the repository hook + `.buildkite/hooks/post-checkout` does the julia checkout swap). + All are plain `buildkite-agent pipeline upload` (no cryptic plugin, no + `cryptic_capable` agent targeting). +2. Record the organization / pipeline / cluster UUIDs that the IAM trust + policies pin (in addition to the slug-based `sub` patterns; slugs can + be renamed or re-minted, UUIDs cannot) in + `ops/terraform/buildkite_ids.auto.tfvars` and commit it (the UUIDs are + not secrets). They come from the Buildkite REST API (token scope + `read_pipelines`): `GET /v2/organizations/julialang` (`.id`) and + `GET /v2/organizations/julialang/pipelines/` (`.id`, + `.cluster_id`). The UUIDs are static; this needs redoing only if a + pipeline is ever recreated or moved to another cluster. Terraform + refuses to apply without real values here. +3. `terraform -chdir=ops/terraform init && terraform -chdir=ops/terraform apply` + — creates the OIDC provider for `agent.buildkite.com`, the four KMS + keys (the notary key as `EXTERNAL`-origin, pending import), and the + IAM roles + policies (one stage role per build pipeline, tokens for + julia-ci only, publish + docs-deploy). Re-apply any time trust + patterns or policies change. Local state is fine while testing (it + contains no secrets); once things settle, move it to S3 by adding a + `backend "s3"` block (bucket `julia-ci-tfstate`, `use_lockfile = + true`) and running `terraform init -migrate-state` — the resources + are untouched by the migration. +4. Key material: + * `./20_export_gpg_pubkey.py` — exports the OpenPGP public half of the + KMS-generated tarball signing key to + `signing-pubkeys/tarball_signing.pub.asc`. Commit it, and publish it as the + new Julia releases signing key (it **replaces** the pre-migration + `juliareleases.asc`; old signatures keep verifying against the old + key, new signatures only against this one). The creation timestamp is + part of the key fingerprint; it defaults to the KMS key's own + immutable CreationDate, so no date need be pinned by hand. + * `./21_import_notary_key.sh AuthKey_X.p8 ` — the + App Store Connect API key is Apple-generated and must be imported + (from a trusted workstation; obtain the .p8 once via the legacy + cryptic agent key, securely delete it afterwards). Writes + `utilities/macos/notary_api_key.json` — commit it; it contains no + secret material. +5. Telemetry tokens into SSM: + * `./23_put_tokens.sh codecov_token` + * `./23_put_tokens.sh coveralls_token` + * `./23_put_tokens.sh buildkite_analytics_token` +6. macOS certificate for the new KMS key: + * `./22_generate_macos_csr.sh` + * Submit CSR at developer.apple.com → Developer ID Application cert + * `openssl x509 -inform DER -in developerID_application.cer -out utilities/macos/developer_id.pem` + and commit (certificates are public). +7. Docs deploy key: + * `./24_docs_deploy_pubkey.sh` and register the printed key as a + deploy key with write access on JuliaLang/docs.julialang.org. + * Ensure the `aws_uploader` rootfs image (JuliaCI/rootfs-images) + ships `aws_kms_pkcs11.so` (https://github.com/JackOfMostTrades/aws-kms-pkcs11). +8. rcodesign (apple-codesign + AWS KMS backend) comes from `rcodesign_jll`, + built by Yggdrasil (`R/rcodesign`) from JuliaCI/apple-platform-rs branch + `julia-build`. To ship a new build: push `julia-build`, bump the + `GitSource` commit in the Yggdrasil recipe, then update the version / + sha256 / URL pins in `utilities/macos/get_rcodesign.sh` + (see `utilities/macos/rcodesign/README.md`). + (The macOS Julia.app is built + staged by the build_ step on the Mac + builder; the publish step only codesigns + packages it, so there is no + committed `.app` skeleton anymore.) +9. `terraform -chdir=ops/terraform/azure apply -var azure_app_id=` + (with Azure credentials that may manage the Trusted Signing app + registration) — federated credentials for Windows Trusted Signing + (matched to the `julia-publish` pipeline, where Windows signing now + runs); fill the (non-secret) `AZURE_TENANT_ID` / `AZURE_CLIENT_ID` + placeholders on the `publish_all` step in `pipelines/publish/launch.yml`. + If flexible federated credentials are unavailable on the tenant, fall + back to `--subject-claim organization_id` tokens (exact-match credential + on the Buildkite organization UUID) at the cost of org-level granularity. +10. Fill `JULIA_CI_AWS_ACCOUNT_ID` in `utilities/aws_oidc.sh` (from the + `julia_ci_aws_account_id` Terraform output). +11. Once green: revoke the legacy static AWS IAM user, delete the cryptic + agent keys from the agents, decommission `cryptic_capable` queues, + turn off webhook builds on the legacy `julia-buildkite` / + `julia-buildkite-scheduled` self-test pipelines (superseded by + `julia-buildkite-ci`) and retire the old `julialang-ephemeral` + self-test bucket, + revoke the old Apple Developer ID certificate, the Apple ID + app-specific password, the old SSH deploy key, and the + `AZURE_CLIENT_SECRET`. Revoke the old GPG release signing key (its + private half lived on agents) and securely delete all copies; keep + its revoked public key published so old releases remain verifiable. + +### NON-PRODUCTION publish test stack (`julia-publish-test-nosecrets`) + +An isolated stack to debug the publish flow (download → KMS-sign → promote) +end to end without touching any production key, bucket, or `julia-latest-*` +pointer. It runs the *same* `publish.sh` but redirects every signer to a +throwaway `*-test` KMS key (self-signed cert) and every bucket to +`julialang-test-publish`, and skips the two signers with no non-production +equivalent (Windows Azure Trusted Signing, macOS Apple notarization). The +macOS `.dmg` is still built and codesigned. All of it lives in +`ops/terraform/test_publish.tf` (gated on `buildkite_test_pipeline_id`) and +`pipelines/publish-test/`. + +1. Create the `julia-publish-test-nosecrets` Buildkite pipeline (WebUI = + `pipelines/publish-test/0_webui.yml`); it is NOT a trust boundary, so PR + builds may stay enabled. Put its pipeline + cluster UUIDs in + `buildkite_ids.auto.tfvars` (`buildkite_test_pipeline_id` / + `buildkite_test_cluster_id`). +2. `terraform apply` — creates the two test keys (`alias/julia-{tarball-signing, + macos-codesigning}-test`), the `julialang-test-publish` bucket, and the + `julia-oidc-publish-test` role (test keys + test bucket only, no write-once). +3. Generate + commit the macOS test cert (AWS creds with kms:Sign on the test + key, plus a built rcodesign): + * `./32_gen_test_codesign_cert.sh` → `utilities/macos/developer_id_test.pem` + The GPG test pubkey is NOT committed: the test pipeline sets + `TARBALL_SIGNING_PUBKEY=""` and derives the key identity from KMS at runtime + (`kms_gpg_sign.py --public-key-from-kms`, using `kms:GetPublicKey` for the + RSA public half and `kms:DescribeKey` for the key's creation date). +4. Stage real input: run a `julia-ci` (or `julia-pr`) build so it stages the + unsigned tarballs (and the unsigned macOS Julia.app) for some commit into + its ephemeral bucket. The test pipeline reads `STAGING_BUCKET` + (`julialang-ephemeral-ci`, public-read) and only PROMOTES to the test + bucket, so no seeding is needed. +5. Trigger a `julia-publish-test-nosecrets` build on that same commit and watch + it download the staged artifacts → KMS-sign (test keys) → build the `.dmg` → + promote into `julialang-test-publish`. Re-run freely (the test role has no + write-once condition). Tear down by deleting `ops/terraform/test_publish.tf`. + +## Agent/rootfs prerequisites + +* AWS CLI on **all build agents/rootfs images** (the build step itself + stages to S3 now), recent enough for + `aws s3api put-object --if-none-match` (conditional writes). For linux + builds that means inside the build rootfs images (`package_*`); also the + macOS / Windows / FreeBSD build agents. +* AWS CLI on test agents/rootfs images too (the test step fetches the + Test Analytics token from SSM itself). This one is soft: test_julia.sh + skips the analytics upload with a warning when `aws` is missing. +* `aws_kms_pkcs11.so` in the `aws_uploader` rootfs (docs deploy). + +### Publish image prerequisites (linux, `queue: publish`) + +The single publish step signs and packages for every OS on linux: + +* AWS CLI; python3 (stdlib only; `kms_gpg_sign.py`, plist editing, PE + signature checks); a host julia (provided by the JuliaCI/julia plugin; + patches pkgimage checksums in the foreign install trees). +* macOS: `rcodesign` is fetched at runtime (pinned sha256, see + `utilities/macos/get_rcodesign.sh`); for the `.dmg`: `mkfs.hfsplus` + (hfsprogs) and the `hfsplus` + `dmg` tools from + [mozilla/libdmg-hfsplus](https://github.com/mozilla/libdmg-hfsplus) + (the same tools Mozilla uses to package Firefox DMGs on linux; the + mozilla fork carries the `symlink` and `attr` subcommands we use). +* Windows: Wine (64-bit) with Inno Setup 6 installed in the prefix at + `C:\Program Files (x86)\Inno Setup 6` (or set `ISCC_EXE`); `jsign` + >= 6.0 on PATH (Azure Trusted Signing storetype) + a JRE; `7z` (p7zip) + for the .zip. + +## Notes + +* The GPG tarball signing key is **generated inside KMS** and never + exists anywhere else; a new public key must therefore be published + (`ops/20_export_gpg_pubkey.py`), and signatures made after the + migration do not verify against the pre-migration `juliareleases.asc`. + Only the notary key uses KMS `EXTERNAL` (BYOK) import, because Apple + generates App Store Connect API keys and we cannot register our own + public key with Apple. +* Retried upload jobs hitting an existing identical object are handled in + `utilities/upload_julia.sh` (412 + ETag comparison), not by allowing + overwrites. +* juliaup needs a change to consume PR binaries: resolve PR number → head + sha via the GitHub API, then fetch + `s3://julialang-ephemeral-pr/bin//julia-*` (this replaces the old + `julia-prNNNN-` upload filenames). diff --git a/ops/common.sh b/ops/common.sh new file mode 100755 index 000000000..9cb3e8bd5 --- /dev/null +++ b/ops/common.sh @@ -0,0 +1,45 @@ +# Shared configuration for the julia-buildkite AWS ops scripts. +# Source this from the numbered scripts; do not run directly. +# +# These scripts cover the imperative one-time operations (key material +# import, secret entry, CSR generation) that deliberately do NOT live in +# Terraform: their inputs are secrets that must never enter the Terraform +# state. The declarative infrastructure (OIDC provider, KMS keys, IAM +# roles, Azure federated credentials) is managed in ops/terraform/. +# +# All scripts are idempotent: safe to re-run after partial failures. + +set -euo pipefail + +# --- Configuration (override via environment) -------------------------------- +# Must match the corresponding Terraform variables (ops/terraform/variables.tf). + +export AWS_REGION="${AWS_REGION:-us-east-1}" + +# S3 bucket that release binaries (and prebuilt CI tools) are uploaded to +export S3_BUCKET="${S3_BUCKET:-julialangnightlies}" + +# KMS key aliases (created by ops/terraform) +export KMS_ALIAS_MACOS_CODESIGN="alias/julia-macos-codesigning" +export KMS_ALIAS_NOTARY_API="alias/julia-notary-api" +export KMS_ALIAS_TARBALL_SIGNING="alias/julia-tarball-signing" +export KMS_ALIAS_DOCS_DEPLOY="alias/julia-docs-deploy" + +# Throwaway NON-PRODUCTION test keys (created by ops/terraform/test_publish.tf +# only when the test pipeline UUID is set). Used by the isolated +# julia-publish-test-nosecrets debug stack with self-signed material. +export KMS_ALIAS_MACOS_CODESIGN_TEST="alias/julia-macos-codesigning-test" +export KMS_ALIAS_TARBALL_SIGNING_TEST="alias/julia-tarball-signing-test" + +# SSM Parameter Store prefix for CI telemetry bearer tokens (codecov, +# coveralls, buildkite analytics). These cannot be public-key operations, +# so they live in the AWS secrets store, fetched at runtime via OIDC. +export SSM_TOKEN_PREFIX="/julia-ci/tokens" + +# --- Helpers ------------------------------------------------------------------ + +# kms_key_arn -> key ARN (empty if alias doesn't exist) +kms_key_arn() { + aws kms describe-key --key-id "$1" --region "${AWS_REGION}" \ + --query KeyMetadata.Arn --output text 2>/dev/null || true +} diff --git a/ops/terraform/.terraform.lock.hcl b/ops/terraform/.terraform.lock.hcl new file mode 100644 index 000000000..e976e31c7 --- /dev/null +++ b/ops/terraform/.terraform.lock.hcl @@ -0,0 +1,47 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.50.0" + constraints = ">= 6.0.0" + hashes = [ + "h1:mNg4vBXXqbO0hY2jCxhOyKVrnjEO0viTG2EY4oAlWaQ=", + "zh:0072806bb262c6d86bc25b4a75750e469881144c14818afdba7b82db840e1588", + "zh:1ebc2dae335dad7a8b16a1985b69a63a14954282bb44fdba7d5103f77551ac7b", + "zh:2dab48fe8f3193b8216d578ac1e3674fa566435cc7dbce2953d55b72e31d0241", + "zh:2fc3d3029c2b7429472391ef339672e1fca8e6ff32c8a519bf3acedafa7e24fe", + "zh:38a36e64e7212f6cedac861ea4d449cce07131b3378de601bf9d49a99e000208", + "zh:3ac70758ed251ce78b7f541a5a79cc6fe56474412783ae1decef719bdd0f30bf", + "zh:4385d3903e685bddb2b8005b4eb7db89f030267d4d03c7d792d2f5e739cc874a", + "zh:4cce0760b87fbafd51f30faec2a737f4183b7c615f4a86557f7d3c893a610dc5", + "zh:4feaeed18694239b896c6415d9a1e5ef89e1da4f4ad60924aa0522adeb1f6599", + "zh:502fca2be1c95f443c3e67d0555601d1de65b4ca82d197c059e9c868360e3a0a", + "zh:57d037f6fdd045f2660909c3bdface9622d81165ce647479cba98d1f353c5eab", + "zh:5dc5a0b915c2ac5256d909458f5c8e40b35f78b3a36ea893c86624eaf6c54e37", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b84c87c58a320adbb2c74a4cad03ae5aac7f2eae21db26f00fdde98c8c4d4523", + "zh:c895f1d5cbcbeff77850ac99efd36bde0048d4e909b296882331b9b9ebf48cfa", + "zh:ead82831683619124597a1f170dd31e9b293e9cf22f558cb166d5e734fcd11e4", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.3.0" + constraints = ">= 4.0.0" + hashes = [ + "h1:j/BqLS2N2AScZyotd9nZpHdieJ7e5S8y+A+ZfIu8kL8=", + "zh:0ab58d6f8991d436c7d2dbd89ed814709b949b07ac5a54ee53b0aec1fa772a8b", + "zh:60b347abcb56f45d97c56f14d895069cd15a83993f199777f571b79fea3642ee", + "zh:6889be32640349230de3f23856e6f04e0e9ced4a84a27d3f552fa54684448218", + "zh:73f8e1ecf7135033165fb14b7e8bf4d656f3ce13065ec35762ea0481975328c7", + "zh:94ce25ee253eca0b42cae9c856b36bca8103b6453012d1b279c3623c805f2d42", + "zh:96bc6de9fd67bc446fd11257872e1ffb1029a996ed1d65a3f6b43f6d408ad9ab", + "zh:97c609a310a51bfd504d704e036d72064a84bf0bdb36cc08cd4cc66098212b41", + "zh:a12c16e94533c5bd123f75032576b9dc91dd5d5ccd5f7cf331d0f2e1adc55cf8", + "zh:c4f014f876adf7af57188795050bda5b0029d8c7d7773031102b6c36dcf1fc21", + "zh:d9b0a21583aaa3df3a95394fb949a3c515ff71c2ff5a1fc4a73d364aa90bfca5", + "zh:da510d22f0c6d71ad19a76406f106b782448f512375787ecfabb338ed1e311a7", + "zh:f0e9447a9ce3a24cdaa113089e65663c836d8b9bfdb915a1c0284e0112cab5c0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/ops/terraform/azure/.terraform.lock.hcl b/ops/terraform/azure/.terraform.lock.hcl new file mode 100644 index 000000000..20700bc7e --- /dev/null +++ b/ops/terraform/azure/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "3.8.0" + constraints = ">= 3.7.0" + hashes = [ + "h1:kIEmiknJFWKe44U9ePG5vXwoloKep0Dbi/mQ8uqCbw0=", + "zh:0d26cfbf9417acd1c2295ccd5b0052abeac85ad1c3f6422ff09bf6a1ce16f00d", + "zh:144d4ea92fed541a6376bc76ad65ba4738dfd7bcab4c9d6cc20d35001338d06d", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:2061d2cb64d8167d0af37e6610d0dc051977bd1ccc0e5cdd5ab02525ee239f96", + "zh:75562fdc3b313b7e538907199bfa588a1fcbc40113b0f3b7bfb496fbc358a32f", + "zh:78bef022ae9b1b0c636b7dd32bcda13fb273023f3a888cc005f3aa20e365b417", + "zh:ad8dbee59843154f8e93b24db9939a4257d13c7c86331eb93f1691294bc4e31f", + "zh:b3d83d7ac57073631704336a188cb746c473f728fc7ccb76abecb520e83fdf65", + "zh:c0bf9e0be73843de9089597be2720e4093b3ba320fbad99ab86da47681e77949", + "zh:c9a4c27d2b0800d3f4ece19d66c1fa574f7cd4ff66277af8f120d65e8f03f48e", + "zh:cd9ad8c848e17d9824045c33132cc0e87aa4d58cdb7bee6c0f6c3f9bc27892d5", + "zh:fbcafc21cdd19451274b905f9ee8a5b758ca63cc231e7544c815642e4a399c6d", + ] +} diff --git a/ops/terraform/azure/main.tf b/ops/terraform/azure/main.tf new file mode 100644 index 000000000..d36c2e2d3 --- /dev/null +++ b/ops/terraform/azure/main.tf @@ -0,0 +1,48 @@ +# Azure workload identity federation so Windows codesigning (Azure Trusted +# Signing via signtool dlib) authenticates with Buildkite OIDC tokens +# instead of an AZURE_CLIENT_SECRET. +# +# This is a separate root module from ../ because it authenticates against +# a different control plane (Microsoft Graph, not AWS); apply it with +# credentials that may manage the Trusted Signing app registration. +# +# Buildkite `sub` claims embed the commit sha, so exact-match federated +# credentials cannot work. Use flexible federated identity credentials +# (claims matching expressions) to wildcard-match protected refs of the +# julia-publish pipeline (Windows codesigning happens in the trusted +# publish step). Requires Microsoft Graph support for flexible FIC on the +# tenant; if unavailable, fall back to `--subject-claim organization_id` +# tokens (exact-match credential on the Buildkite organization UUID) at +# the cost of org-level granularity -- see ops/README.md. + +locals { + bk_oidc_host = "agent.buildkite.com" + + # Must stay in sync with publish_sub_patterns in ../iam.tf. + publish_sub_patterns = { + master = "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/master:*" + release = "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/release-*:*" + tags = "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/tags/v*:*" + } +} + +data "azuread_application" "trusted_signing" { + client_id = var.azure_app_id +} + +resource "azuread_application_flexible_federated_identity_credential" "buildkite" { + for_each = local.publish_sub_patterns + + application_id = data.azuread_application.trusted_signing.id + display_name = "buildkite-julia-publish-${each.key}" + description = "Buildkite julia-publish (${each.key}) Windows Trusted Signing" + audience = "api://AzureADTokenExchange" + issuer = "https://${local.bk_oidc_host}" + claims_matching_expression = "claims['sub'] matches '${each.value}'" +} + +# Publish jobs authenticate via: +# buildkite-agent oidc request-token --audience api://AzureADTokenExchange +# with AZURE_CLIENT_ID=, AZURE_TENANT_ID, and +# AZURE_FEDERATED_TOKEN_FILE set by utilities/windows/codesign.sh. +# Remove any AZURE_CLIENT_SECRET from the app once this works. diff --git a/ops/terraform/azure/variables.tf b/ops/terraform/azure/variables.tf new file mode 100644 index 000000000..8d903aca9 --- /dev/null +++ b/ops/terraform/azure/variables.tf @@ -0,0 +1,10 @@ +variable "azure_app_id" { + description = "Client ID of the existing app registration that holds the Trusted Signing 'Code Signing Certificate Profile Signer' role assignment" + type = string +} + +variable "bk_org" { + description = "Buildkite organization slug" + type = string + default = "julialang" +} diff --git a/ops/terraform/azure/versions.tf b/ops/terraform/azure/versions.tf new file mode 100644 index 000000000..b96c61266 --- /dev/null +++ b/ops/terraform/azure/versions.tf @@ -0,0 +1,13 @@ +terraform { + # azuread >= 3.7 is needed for + # azuread_application_flexible_federated_identity_credential. + required_version = ">= 1.5" + required_providers { + azuread = { + source = "hashicorp/azuread" + version = ">= 3.7" + } + } +} + +provider "azuread" {} diff --git a/ops/terraform/buildkite_ids.auto.tfvars b/ops/terraform/buildkite_ids.auto.tfvars new file mode 100644 index 000000000..0ef22b3a0 --- /dev/null +++ b/ops/terraform/buildkite_ids.auto.tfvars @@ -0,0 +1,31 @@ +# Buildkite UUIDs pinned by the IAM trust policies (see variables.tf). +# Not secrets; fetched 2026-06-12 from the Buildkite REST API. These are +# static -- refetch only if a pipeline is ever recreated or moved between +# clusters (and re-apply). +# +# julia-publish runs in its own cluster, separate from the build +# pipelines: the cluster_id condition then also stops a compromised +# build-pool agent from minting publish-shaped tokens. + +buildkite_organization_id = "d409823c-5fa7-41c8-9033-7269c5fde4f3" + +buildkite_pipeline_ids = { + "julia-pr" = "019ebd61-5b1f-428e-b08c-1b5a2111e001" + "julia-ci" = "019ebd63-635e-4155-b60a-9d2815900786" + "julia-publish" = "019ebd63-df36-4f53-a07f-4b31064df0f8" + "julia-buildkite-ci" = "019f3d33-d54f-4087-b70f-767d53b801d4" +} + +buildkite_cluster_ids = { + "julia-pr" = "ae7e6bd1-fde8-433d-bac7-9d2d01108ed6" + "julia-ci" = "ae7e6bd1-fde8-433d-bac7-9d2d01108ed6" + "julia-buildkite-ci" = "ae7e6bd1-fde8-433d-bac7-9d2d01108ed6" + "julia-publish" = "fd6c2af4-60c1-40ee-bdd5-88ecb6698fbc" +} + +# Isolated NON-PRODUCTION publish test stack (ops/terraform/test_publish.tf). +# Set to the real UUIDs to enable the throwaway test keys/bucket/role; comment +# out (or set null) to disable. julia-publish-test-nosecrets lives in the same +# Secure cluster as julia-publish. +buildkite_test_pipeline_id = "019ec73b-f8df-428a-a256-745eff852687" +buildkite_test_cluster_id = "fd6c2af4-60c1-40ee-bdd5-88ecb6698fbc" diff --git a/ops/terraform/iam.tf b/ops/terraform/iam.tf new file mode 100644 index 000000000..2e0452067 --- /dev/null +++ b/ops/terraform/iam.tf @@ -0,0 +1,392 @@ +# IAM roles assumed by Buildkite jobs via OIDC. +# +# Trust is split into an untrusted tier and a trusted tier so that pull +# request builds can never reach signing keys or release locations: +# +# The untrusted roles exist once per build pipeline -- julia-pr and +# julia-ci never share a role, so a role's identity IS its pipeline: +# +# julia-oidc-stage-pr UNTRUSTED. Assumable from any job of julia-pr +# julia-oidc-stage-ci (resp. julia-ci); the build step assumes it to +# stage its tarball directly. May only write unsigned +# artifacts, write-once, to a path gated by the build's +# own commit sha, inside that one pipeline's OWN +# ephemeral staging bucket (publish only ever reads +# julia-ci's). No KMS, no final-location write. Because +# the permissions are harmless, the spoofable `ref` +# component of the sub claim does not matter here. +# julia-oidc-stage-buildkite UNTRUSTED. Same shape for the +# julia-buildkite repository's own self-test pipeline +# (julia-buildkite-ci), which builds julia with +# proposed pipeline code: write-once staging to its +# own bucket, which nothing downstream consumes. +# julia-oidc-tokens-ci Telemetry bearer tokens from SSM; julia-ci ONLY. +# PR builds get no tokens whatsoever: a malicious pull +# request executes code inside the job and could read +# and exfiltrate any bearer token available to it. +# julia-oidc-publish TRUSTED. kms:Sign with the signing keys, read the +# julia-ci staging bucket, and write the final release +# locations. Assumable ONLY from the `julia-publish` +# pipeline slug. That pipeline must have pull-request +# builds DISABLED and be branch-limited to +# master/release-*/v* (see ops/README.md). Since a PR +# can never produce a build under that slug, the slug +# is the trust boundary -- not the (PR-spoofable) +# branch name. +# julia-oidc-docs-deploy TRUSTED. kms:Sign with the docs SSH key; publish +# pipeline only. +# +# Defense in depth: trusted publish jobs additionally run +# utilities/verify_trusted_commit.sh before assuming their role, which +# refuses unless BUILDKITE_COMMIT is reachable from a protected ref of the +# canonical JuliaLang/julia repository. +# +# Overwrite protection: every PutObject must use the S3 conditional write +# header (s3:if-none-match = "*"), i.e. uploads fail if the object already +# exists. The only exception is `julia-latest-*` pointer objects, which the +# publish pipeline intentionally repoints. + +data "aws_caller_identity" "current" {} + +locals { + # `sub` format: organization:ORG:pipeline:PIPELINE:ref:REF:commit:SHA:step:STEP + + # The UNTRUSTED build pipelines (short suffix used in role names): + # julia-pr builds pull requests + # julia-ci builds trusted refs (master / release-* / tags), incl. scheduled + # Any ref is allowed: the stage/token roles these map to are deliberately + # harmless, so we do not (and must not) rely on the ref component for trust. + build_pipelines = { + "julia-pr" = "pr" + "julia-ci" = "ci" + # the julia-buildkite repository's own self-test CI + "julia-buildkite-ci" = "buildkite" + } + + # The TRUSTED publish pipeline. julia-publish MUST be configured in + # Buildkite with pull-request builds disabled and branch-limited to the + # protected refs (see ops/README.md), and is triggered only by julia-ci. + # A PR can then never produce a build under this slug, so the slug itself + # is the trust boundary; the ref patterns below are belt-and-braces only. + publish_sub_patterns = [ + "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/master:*", + "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/heads/release-*:*", + "organization:${var.bk_org}:pipeline:julia-publish:ref:refs/tags/v*:*", + ] + + # Per-role trust: which pipeline (by slug pattern in `sub` AND by + # unforgeable UUID session tag) and -- for the trusted roles -- which + # steps (aws:RequestTag/step_key, attested by the agent) may assume it. + # Every role trusts exactly ONE pipeline. The untrusted roles have no + # step restriction (null): build steps stage artifacts directly, test + # steps post results directly, and the permissions are deliberately + # harmless anyway. + oidc_trust = { + stage-pr = { + pipelines = ["julia-pr"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-pr:*"] + step_key_patterns = null + } + stage-ci = { + pipelines = ["julia-ci"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-ci:*"] + step_key_patterns = null + } + stage-buildkite = { + pipelines = ["julia-buildkite-ci"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-buildkite-ci:*"] + step_key_patterns = null + } + # NB: there is deliberately no tokens-pr. A pull request executes + # attacker-controlled code inside the job, which could exfiltrate any + # bearer token the job can read -- so PR builds get NO tokens at all. + tokens-ci = { + pipelines = ["julia-ci"] + sub_patterns = ["organization:${var.bk_org}:pipeline:julia-ci:*"] + step_key_patterns = null + } + publish = { + pipelines = ["julia-publish"] + sub_patterns = local.publish_sub_patterns + step_key_patterns = ["publish_*"] + } + docs-deploy = { + pipelines = ["julia-publish"] + sub_patterns = local.publish_sub_patterns + step_key_patterns = ["deploy_docs"] + } + } + + # Allowed cluster UUIDs per role, from the per-pipeline cluster map; + # roles whose pipelines have no configured cluster get no condition. + trust_cluster_ids = { + for role, t in local.oidc_trust : + role => distinct(compact([for p in t.pipelines : lookup(var.buildkite_cluster_ids, p, "")])) + } + + # Artifact-family prefixes inside each staging bucket (normal release / + # no-GPL; see S3_BUCKET_PREFIX selection in utilities/build_envs.sh) + # and the final release locations. + staging_prefixes = [ + var.s3_bucket_prefix, + var.s3_nogpl_prefix, + ] + staging_bucket_arns = { + for slug, bucket in var.s3_staging_buckets : slug => "arn:aws:s3:::${bucket}" + } + final_paths = [ + "arn:aws:s3:::${var.s3_bucket}/${var.s3_bucket_prefix}", + "arn:aws:s3:::${var.s3_nogpl_bucket}/${var.s3_nogpl_prefix}", + ] +} + +data "aws_iam_policy_document" "trust" { + for_each = local.oidc_trust + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity", "sts:TagSession"] + + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.buildkite.arn] + } + + condition { + test = "StringEquals" + variable = "${local.bk_oidc_host}:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringLike" + variable = "${local.bk_oidc_host}:sub" + values = each.value.sub_patterns + } + dynamic "condition" { + for_each = each.value.step_key_patterns != null ? [1] : [] + content { + test = "StringLike" + variable = "aws:RequestTag/step_key" + values = each.value.step_key_patterns + } + } + + # Pin the Buildkite UUIDs, not just the slugs in `sub`: slugs are + # renameable/re-mintable, UUIDs are not. These arrive as session tags + # (utilities/aws_oidc.sh) since IAM cannot condition on other claims. + # A token requested without these tags fails the conditions outright. + condition { + test = "StringEquals" + variable = "aws:RequestTag/organization_id" + values = [var.buildkite_organization_id] + } + condition { + test = "StringEquals" + variable = "aws:RequestTag/pipeline_id" + values = [for p in each.value.pipelines : var.buildkite_pipeline_ids[p]] + } + dynamic "condition" { + for_each = length(local.trust_cluster_ids[each.key]) > 0 ? [1] : [] + content { + test = "StringEquals" + variable = "aws:RequestTag/cluster_id" + values = local.trust_cluster_ids[each.key] + } + } + } +} + +# ---- julia-oidc-stage-{pr,ci} (UNTRUSTED, one role per build pipeline) ------ +# Each role is assumable from exactly one pipeline (trust policy) and can +# write-once to exactly that pipeline's staging bucket, only below the +# build's own commit sha (the build_commit tag is attested by Buildkite, +# not settable by the job). julia-pr's role never mentions the julia-ci +# bucket that publish reads from. + +resource "aws_iam_role" "stage" { + for_each = local.build_pipelines + + name = "julia-oidc-stage-${each.value}" + description = "Buildkite ${each.key} staging upload: write-once to ${var.s3_staging_buckets[each.key]}, own commit path (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["stage-${each.value}"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "stage" { + for_each = local.build_pipelines + + statement { + sid = "WriteOnceToOwnCommitStagingPath" + # No PutObjectAcl: the staging buckets disable ACLs (BucketOwnerEnforced, + # public read via bucket policy), and the s3:if-none-match condition key + # only exists for the PutObject action anyway. + actions = ["s3:PutObject"] + # $${...} is the literal IAM policy variable, not Terraform interpolation. + resources = [ + for p in local.staging_prefixes : + "${local.staging_bucket_arns[each.key]}/${p}/$${aws:PrincipalTag/build_commit}/*" + ] + + condition { + test = "StringEquals" + variable = "s3:if-none-match" + values = ["*"] + } + } + + statement { + sid = "ReadStagingForRetryChecks" + actions = ["s3:GetObject"] + resources = ["${local.staging_bucket_arns[each.key]}/*"] + } +} + +resource "aws_iam_role_policy" "stage" { + for_each = local.build_pipelines + + name = "write-once-uploads-to-own-commit-staging-path" + role = aws_iam_role.stage[each.key].id + policy = data.aws_iam_policy_document.stage[each.key].json +} + +# ---- julia-oidc-publish (TRUSTED) ------------------------------------------- +# Sign + promote staged artifacts to the final release locations. Assumable +# only from the julia-publish pipeline slug (PR builds disabled there). + +resource "aws_iam_role" "publish" { + name = "julia-oidc-publish" + description = "Buildkite publish: sign + promote staged artifacts to release (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["publish"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "publish" { + statement { + sid = "ReadStagedArtifacts" + actions = ["s3:GetObject"] + # Only the julia-ci staging bucket: artifacts staged by pull-request + # builds (julia-pr's bucket) are not publishable inputs. + resources = ["${local.staging_bucket_arns["julia-ci"]}/*"] + } + + statement { + sid = "WriteOnceToFinalLocations" + actions = ["s3:PutObject"] + resources = [for p in local.final_paths : "${p}/*"] + + condition { + test = "StringEquals" + variable = "s3:if-none-match" + values = ["*"] + } + } + + # Versioned objects are uploaded public-read (--acl). The s3:if-none-match + # condition key exists only for s3:PutObject, so a PutObjectAcl folded into + # the conditioned statement above would never match -- keep it unconditioned. + # Object content stays write-once: that is enforced by the PutObject grant. + statement { + sid = "SetPublicReadAclOnFinalObjects" + actions = ["s3:PutObjectAcl"] + resources = [for p in local.final_paths : "${p}/*"] + } + + statement { + sid = "RepointLatestPointers" + actions = ["s3:PutObject", "s3:PutObjectAcl"] + resources = flatten([for p in local.final_paths : [ + "${p}/*/julia-latest-*", + "${p}/*/*/julia-latest-*", + "${p}/*/*/*/julia-latest-*", + ]]) + } + + statement { + sid = "ReadFinalForRetryChecks" + actions = ["s3:GetObject"] + resources = [for p in local.final_paths : "${p}/*"] + } + + statement { + sid = "ReleaseSigning" + actions = ["kms:Sign", "kms:GetPublicKey"] + resources = [ + aws_kms_key.macos_codesign.arn, + aws_kms_external_key.notary_api.arn, + aws_kms_key.tarball_signing.arn, + ] + + condition { + test = "StringLike" + variable = "aws:PrincipalTag/step_key" + values = ["publish_*"] + } + } +} + +resource "aws_iam_role_policy" "publish" { + name = "sign-and-promote-staged-artifacts-to-release" + role = aws_iam_role.publish.id + policy = data.aws_iam_policy_document.publish.json +} + +# ---- julia-oidc-docs-deploy (TRUSTED) --------------------------------------- + +resource "aws_iam_role" "docs_deploy" { + name = "julia-oidc-docs-deploy" + description = "Buildkite docs deploy: SSH signing via KMS (aws-kms-pkcs11, via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["docs-deploy"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "docs_deploy" { + statement { + sid = "SshSignViaDocsDeployKey" + actions = ["kms:Sign", "kms:GetPublicKey"] + resources = [aws_kms_key.docs_deploy.arn] + + condition { + test = "StringEquals" + variable = "aws:PrincipalTag/step_key" + values = ["deploy_docs"] + } + } + # No S3 statement is needed to fetch the staged HTML docs: the staging bucket + # grants public s3:GetObject via its bucket policy, and deploy_docs.yml reads + # the htmldocs by its exact key (no ListBucket / --recursive). +} + +resource "aws_iam_role_policy" "docs_deploy" { + name = "ssh-sign-docs-deploy-via-kms" + role = aws_iam_role.docs_deploy.id + policy = data.aws_iam_policy_document.docs_deploy.json +} + +# ---- julia-oidc-tokens-ci ---------------------------------------------------- +# CI telemetry bearer tokens (codecov, coveralls, buildkite analytics) are +# inherently symmetric secrets, so they live in SSM Parameter Store +# (SecureString, see ops/23_put_tokens.sh) and are fetched at runtime by +# this role. Nothing secret is stored in the repository or this state. +# julia-ci ONLY -- pull request builds run attacker-controlled code and +# therefore get no bearer tokens at all (there is no tokens-pr role). + +resource "aws_iam_role" "tokens" { + name = "julia-oidc-tokens-ci" + description = "Buildkite julia-ci telemetry: read coverage/analytics tokens from SSM (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.trust["tokens-ci"].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "tokens" { + statement { + sid = "ReadTelemetryTokens" + actions = ["ssm:GetParameter"] + resources = ["arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm_token_prefix}/*"] + } +} + +resource "aws_iam_role_policy" "tokens" { + name = "read-telemetry-tokens-from-ssm" + role = aws_iam_role.tokens.id + policy = data.aws_iam_policy_document.tokens.json +} diff --git a/ops/terraform/kms.tf b/ops/terraform/kms.tf new file mode 100644 index 000000000..4ed12efe1 --- /dev/null +++ b/ops/terraform/kms.tf @@ -0,0 +1,101 @@ +# The KMS keys used by Julia CI. No private key ever exists outside KMS +# (or, for the BYOK notary key, outside the one-time trusted import +# machine). +# +# The EXTERNAL-origin notary key is created here *without* key material; +# import the existing .p8 out-of-band (it must never enter the Terraform +# state) with: +# +# ops/21_import_notary_key.sh -> alias/julia-notary-api +# +# Until then it sits in the PendingImport state and cannot sign. + +# Fresh key; a new Developer ID certificate must be issued for it +# (see ops/22_generate_macos_csr.sh). +resource "aws_kms_key" "macos_codesign" { + description = "Julia macOS Developer ID codesigning key (used via rcodesign)" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = "RSA_2048" + + # The signing keys are the only unrecoverable resources in this module + # (the GPG key exists nowhere else; the notary .p8 is destroyed after + # import; the others are bound to issued certificates / registered + # deploy keys). prevent_destroy makes any plan that would destroy or + # REPLACE one (key specs are immutable, so e.g. a key_spec edit plans + # as replacement) a hard error. 30 days is the maximum (and default) + # cancellation window for a scheduled deletion; set explicitly. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "macos_codesign" { + name = "alias/julia-macos-codesigning" + target_key_id = aws_kms_key.macos_codesign.key_id +} + +# The existing App Store Connect API .p8 key is imported (Apple generates +# ASC API keys; we cannot register our own public key with Apple). +resource "aws_kms_external_key" "notary_api" { + description = "Julia App Store Connect API key (notarization JWT signing)" + key_usage = "SIGN_VERIFY" + key_spec = "ECC_NIST_P256" + + # See the lifecycle note on aws_kms_key.macos_codesign. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "notary_api" { + name = "alias/julia-notary-api" + target_key_id = aws_kms_external_key.notary_api.id +} + +# Fresh key generated inside KMS; the OpenPGP public key for it is +# exported with ops/20_export_gpg_pubkey.py and must be published as the +# new Julia releases signing key (signatures do NOT verify against the +# pre-migration juliareleases.asc). +resource "aws_kms_key" "tarball_signing" { + description = "Julia release tarball GPG signing key" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = var.tarball_key_spec + + # See the lifecycle note on aws_kms_key.macos_codesign. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "tarball_signing" { + name = "alias/julia-tarball-signing" + target_key_id = aws_kms_key.tarball_signing.key_id +} + +# The SSH key used to push to docs.julialang.org. SSH signs via the +# aws-kms-pkcs11 provider, so the key never exists outside KMS. Register +# the public key (ops/24_docs_deploy_pubkey.sh) as a GitHub deploy key +# with write access. +resource "aws_kms_key" "docs_deploy" { + description = "Julia docs.julialang.org SSH deploy key (used via aws-kms-pkcs11)" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = "ECC_NIST_P256" + + # See the lifecycle note on aws_kms_key.macos_codesign. + deletion_window_in_days = 30 + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_kms_alias" "docs_deploy" { + name = "alias/julia-docs-deploy" + target_key_id = aws_kms_key.docs_deploy.key_id +} diff --git a/ops/terraform/oidc.tf b/ops/terraform/oidc.tf new file mode 100644 index 000000000..271a4fafc --- /dev/null +++ b/ops/terraform/oidc.tf @@ -0,0 +1,22 @@ +# IAM OIDC identity provider for Buildkite agents. +# +# Once this exists, Buildkite jobs can exchange `buildkite-agent oidc +# request-token --audience sts.amazonaws.com` tokens for temporary AWS +# credentials via sts:AssumeRoleWithWebIdentity (no static secrets). + +locals { + bk_oidc_host = "agent.buildkite.com" +} + +# AWS now validates most OIDC issuers against trusted root CAs and ignores +# the thumbprint, but the API still requires one: use the root certificate +# of the issuer's chain. +data "tls_certificate" "buildkite" { + url = "https://${local.bk_oidc_host}" +} + +resource "aws_iam_openid_connect_provider" "buildkite" { + url = "https://${local.bk_oidc_host}" + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = [data.tls_certificate.buildkite.certificates[0].sha1_fingerprint] +} diff --git a/ops/terraform/outputs.tf b/ops/terraform/outputs.tf new file mode 100644 index 000000000..f31d9b2de --- /dev/null +++ b/ops/terraform/outputs.tf @@ -0,0 +1,35 @@ +output "oidc_provider_arn" { + description = "IAM OIDC provider for agent.buildkite.com" + value = aws_iam_openid_connect_provider.buildkite.arn +} + +output "role_arns" { + description = "IAM roles assumed by Buildkite jobs" + value = { + stage-pr = aws_iam_role.stage["julia-pr"].arn + stage-ci = aws_iam_role.stage["julia-ci"].arn + tokens-ci = aws_iam_role.tokens.arn + publish = aws_iam_role.publish.arn + docs-deploy = aws_iam_role.docs_deploy.arn + } +} + +output "kms_key_arns" { + description = "KMS signing keys (the EXTERNAL notary key needs material imported)" + value = { + macos_codesign = aws_kms_key.macos_codesign.arn + notary_api = aws_kms_external_key.notary_api.arn + tarball_signing = aws_kms_key.tarball_signing.arn + docs_deploy = aws_kms_key.docs_deploy.arn + } +} + +output "staging_buckets" { + description = "Per-pipeline ephemeral staging buckets (PR consumers like juliaup read the julia-pr one)" + value = { for slug, b in aws_s3_bucket.staging : slug => b.bucket } +} + +output "julia_ci_aws_account_id" { + description = "Fill this into JULIA_CI_AWS_ACCOUNT_ID in utilities/aws_oidc.sh" + value = data.aws_caller_identity.current.account_id +} diff --git a/ops/terraform/s3.tf b/ops/terraform/s3.tf new file mode 100644 index 000000000..196a6e69a --- /dev/null +++ b/ops/terraform/s3.tf @@ -0,0 +1,73 @@ +# The per-pipeline ephemeral staging buckets. Everything in them is +# unsigned, write-once (enforced by IAM, see iam.tf), keyed by source +# commit sha, publicly readable (juliaup fetches PR binaries from the +# julia-pr bucket anonymously), and expired by lifecycle policy. +# +# The release buckets (julialangnightlies, julialang-nogpl) predate this +# module and are not managed here. + +resource "aws_s3_bucket" "staging" { + for_each = var.s3_staging_buckets + + bucket = each.value +} + +# Object ACLs are disabled entirely (the modern S3 arrangement): public +# readability comes from the bucket policy below, and uploads must NOT +# send --acl (an ACL'd PUT would both fail under BucketOwnerEnforced and +# require s3:PutObjectAcl, which the stage roles do not have -- the +# s3:if-none-match condition key only exists for the PutObject action). +resource "aws_s3_bucket_ownership_controls" "staging" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_public_access_block" "staging" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false +} + +# Everything staged is world-readable: PR binaries are consumed by juliaup +# anonymously, and the artifacts are public release candidates anyway. +resource "aws_s3_bucket_policy" "staging_public_read" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + depends_on = [aws_s3_bucket_public_access_block.staging] + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "PublicReadStagedArtifacts" + Effect = "Allow" + Principal = "*" + Action = "s3:GetObject" + Resource = "${each.value.arn}/*" + }] + }) +} + +resource "aws_s3_bucket_lifecycle_configuration" "staging" { + for_each = aws_s3_bucket.staging + + bucket = each.value.id + rule { + id = "expire-staged-artifacts" + status = "Enabled" + filter { + prefix = "" + } + expiration { + days = var.staging_expiry_days[each.key] + } + } +} diff --git a/ops/terraform/test_publish.tf b/ops/terraform/test_publish.tf new file mode 100644 index 000000000..501fa29b0 --- /dev/null +++ b/ops/terraform/test_publish.tf @@ -0,0 +1,242 @@ +# Isolated, NON-PRODUCTION publish test stack for `julia-publish-test-nosecrets`. +# +# This duplicates just enough of the real publish stack to exercise the full +# download -> KMS-sign -> promote flow end to end, but against THROWAWAY KMS +# keys (with self-signed certs) and a SEPARATE bucket -- so the publish flow +# can be debugged freely without touching any production key, bucket, or the +# public `julia-latest-*` pointers. +# +# Everything here is self-contained in this one file and gated on +# `var.buildkite_test_pipeline_id` being set: a normal `terraform apply` +# (test pipeline UUID unset) creates none of it. To tear the test stack down, +# delete this file (the keys have no prevent_destroy) -- production is untouched. +# +# Deliberately omitted (per design): Windows Authenticode signing (Azure) and +# macOS notarization (Apple ASC) have no non-production equivalent, so the test +# pipeline skips them (PUBLISH_SKIP_WINDOWS_SIGN / PUBLISH_SKIP_NOTARIZATION). +# Hence only two test keys: GPG tarball signing and macOS codesigning. + +variable "buildkite_test_pipeline_id" { + description = "UUID of the julia-publish-test-nosecrets Buildkite pipeline (set in buildkite_ids.auto.tfvars to enable the test stack)" + type = string + default = null + + validation { + condition = var.buildkite_test_pipeline_id == null ? true : can(regex("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.buildkite_test_pipeline_id)) && var.buildkite_test_pipeline_id != "00000000-0000-0000-0000-000000000000" + error_message = "buildkite_test_pipeline_id must be the pipeline's UUID (or null to disable the test stack)." + } +} + +variable "buildkite_test_cluster_id" { + description = "UUID of the cluster the julia-publish-test-nosecrets pipeline runs in (optional; omit for no cluster condition)" + type = string + default = null +} + +variable "s3_test_publish_bucket" { + description = "Bucket for the isolated publish test stack (staging input + promotion output)" + type = string + default = "julialang-test-publish" +} + +locals { + # The whole test stack materializes only once the pipeline UUID is supplied. + enable_test_publish = var.buildkite_test_pipeline_id != null + test_count = local.enable_test_publish ? 1 : 0 + # Cluster condition only if a test cluster UUID was given. + test_cluster_ids = compact([var.buildkite_test_cluster_id == null ? "" : var.buildkite_test_cluster_id]) +} + +# ---- Throwaway test signing keys (no prevent_destroy) ----------------------- +# Same specs as production (kms.tf): RSA so kms_gpg_sign.py's OpenPGP assembly +# works, RSA_2048 for the macOS codesign key. Public halves become self-signed +# material committed to the repo (ops/20_export_gpg_pubkey.py and +# ops/32_gen_test_codesign_cert.sh). + +resource "aws_kms_key" "tarball_signing_test" { + count = local.test_count + description = "TEST (non-production) Julia release tarball GPG signing key" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = var.tarball_key_spec + deletion_window_in_days = 7 +} + +resource "aws_kms_alias" "tarball_signing_test" { + count = local.test_count + name = "alias/julia-tarball-signing-test" + target_key_id = aws_kms_key.tarball_signing_test[0].key_id +} + +resource "aws_kms_key" "macos_codesign_test" { + count = local.test_count + description = "TEST (non-production) Julia macOS codesigning key (self-signed cert, used via rcodesign)" + key_usage = "SIGN_VERIFY" + customer_master_key_spec = "RSA_2048" + deletion_window_in_days = 7 +} + +resource "aws_kms_alias" "macos_codesign_test" { + count = local.test_count + name = "alias/julia-macos-codesigning-test" + target_key_id = aws_kms_key.macos_codesign_test[0].key_id +} + +# ---- Test bucket (staging input + promotion output) ------------------------- +# One bucket used for both: seeded staging at bin//... and promotion at +# bin///... (distinct key shapes). Mirrors the ephemeral-bucket setup +# (s3.tf): ACLs off, public-read via policy, short lifecycle expiry. + +resource "aws_s3_bucket" "test_publish" { + count = local.test_count + bucket = var.s3_test_publish_bucket +} + +resource "aws_s3_bucket_ownership_controls" "test_publish" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_public_access_block" "test_publish" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false +} + +resource "aws_s3_bucket_policy" "test_publish_public_read" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + depends_on = [aws_s3_bucket_public_access_block.test_publish] + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "PublicReadTestArtifacts" + Effect = "Allow" + Principal = "*" + Action = "s3:GetObject" + Resource = "${aws_s3_bucket.test_publish[0].arn}/*" + }] + }) +} + +resource "aws_s3_bucket_lifecycle_configuration" "test_publish" { + count = local.test_count + bucket = aws_s3_bucket.test_publish[0].id + rule { + id = "expire-test-artifacts" + status = "Enabled" + filter { + prefix = "" + } + expiration { + days = 7 + } + } +} + +# ---- Test OIDC role + standalone trust -------------------------------------- +# Trust is intentionally LOOSE on ref (any ref of the test pipeline slug) so we +# can test from any branch -- but still pinned to the unforgeable org_id + test +# pipeline_id (+ optional cluster_id) session tags, so ONLY the real test +# pipeline can assume it. The role can sign with the test keys and read/write +# ONLY the test bucket. No write-once condition, so the same commit can be +# re-published repeatedly while debugging. + +data "aws_iam_policy_document" "publish_test_trust" { + count = local.test_count + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity", "sts:TagSession"] + + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.buildkite.arn] + } + + condition { + test = "StringEquals" + variable = "${local.bk_oidc_host}:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringLike" + variable = "${local.bk_oidc_host}:sub" + values = ["organization:${var.bk_org}:pipeline:julia-publish-test-nosecrets:*"] + } + condition { + test = "StringEquals" + variable = "aws:RequestTag/organization_id" + values = [var.buildkite_organization_id] + } + condition { + test = "StringEquals" + variable = "aws:RequestTag/pipeline_id" + values = [var.buildkite_test_pipeline_id] + } + dynamic "condition" { + for_each = length(local.test_cluster_ids) > 0 ? [1] : [] + content { + test = "StringEquals" + variable = "aws:RequestTag/cluster_id" + values = local.test_cluster_ids + } + } + } +} + +resource "aws_iam_role" "publish_test" { + count = local.test_count + name = "julia-oidc-publish-test" + description = "NON-PRODUCTION publish test: sign with test KMS keys + read/write the test bucket (via OIDC)" + assume_role_policy = data.aws_iam_policy_document.publish_test_trust[0].json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "publish_test" { + count = local.test_count + + # Read seeded staging input AND read-back for retry checks; write promoted + # artifacts. No s3:if-none-match: overwrites are allowed so the same build + # can be re-run freely. Scoped to the test bucket only. + statement { + sid = "ReadTestBucket" + actions = ["s3:GetObject"] + resources = ["${aws_s3_bucket.test_publish[0].arn}/*"] + } + statement { + sid = "WriteTestBucket" + actions = ["s3:PutObject"] + resources = ["${aws_s3_bucket.test_publish[0].arn}/*"] + } + + statement { + sid = "TestReleaseSigning" + # DescribeKey: kms_gpg_sign.py --public-key-from-kms reads the GPG key's + # CreationDate (part of the OpenPGP fingerprint) so the test pipeline need + # not commit a pubkey. GetPublicKey: fetch the RSA public half. + actions = ["kms:Sign", "kms:GetPublicKey", "kms:DescribeKey"] + resources = [ + aws_kms_key.macos_codesign_test[0].arn, + aws_kms_key.tarball_signing_test[0].arn, + ] + } +} + +resource "aws_iam_role_policy" "publish_test" { + count = local.test_count + name = "test-sign-and-promote-to-test-bucket" + role = aws_iam_role.publish_test[0].id + policy = data.aws_iam_policy_document.publish_test[0].json +} + +output "test_publish_role_arn" { + description = "ARN of the non-production publish test role (null until the test pipeline UUID is set)" + value = local.enable_test_publish ? aws_iam_role.publish_test[0].arn : null +} diff --git a/ops/terraform/variables.tf b/ops/terraform/variables.tf new file mode 100644 index 000000000..3c20443f0 --- /dev/null +++ b/ops/terraform/variables.tf @@ -0,0 +1,142 @@ +variable "aws_region" { + description = "Region holding the KMS keys and SSM parameters" + type = string + default = "us-east-1" +} + +variable "bk_org" { + description = "Buildkite organization slug" + type = string + default = "julialang" +} + +# The Buildkite UUIDs pinned by the IAM trust policies, alongside the +# slug-based `sub` patterns. Slugs are renameable and can be re-minted by +# deleting + recreating a pipeline; UUIDs cannot, so a recreated pipeline +# with a matching slug does not regain role access. The jobs pass these as +# AWS session tags (see utilities/aws_oidc.sh) because IAM can only +# condition on aud/sub from the raw token. Values live in +# buildkite_ids.auto.tfvars (fetched once from the Buildkite REST API: +# GET /v2/organizations/ and /v2/organizations//pipelines/; +# re-fetch only if a pipeline is ever recreated or moved between clusters). +# No defaults and nullable = false: an apply without real, well-formed +# UUIDs (set in buildkite_ids.auto.tfvars) must be impossible -- a trust +# policy missing these conditions would fall back to slug-only pinning. +variable "buildkite_organization_id" { + description = "UUID of the Buildkite organization" + type = string + nullable = false + + validation { + # The null guard keeps `terraform validate` (which leaves required + # variables unset) happy; nullable = false rejects null at plan time. + condition = var.buildkite_organization_id == null ? true : can(regex("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.buildkite_organization_id)) && var.buildkite_organization_id != "00000000-0000-0000-0000-000000000000" + error_message = "Set buildkite_organization_id to the organization UUID (see buildkite_ids.auto.tfvars)." + } +} + +variable "buildkite_pipeline_ids" { + description = "UUIDs of the four Buildkite pipelines, keyed by slug" + type = map(string) + nullable = false + + validation { + condition = var.buildkite_pipeline_ids == null ? true : keys(var.buildkite_pipeline_ids) == tolist(["julia-buildkite-ci", "julia-ci", "julia-pr", "julia-publish"]) + error_message = "buildkite_pipeline_ids must have exactly the keys julia-buildkite-ci, julia-ci, julia-pr, julia-publish." + } + validation { + condition = var.buildkite_pipeline_ids == null ? true : alltrue([ + for id in values(var.buildkite_pipeline_ids) : + can(regex("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", id)) && id != "00000000-0000-0000-0000-000000000000" + ]) + error_message = "Set every pipeline UUID (see buildkite_ids.auto.tfvars)." + } +} + +# Cluster UUID each pipeline runs in, keyed by pipeline slug. Optional: +# pipelines without an entry get no cluster condition (e.g. unclustered +# agents, where the cluster_id claim is empty). +variable "buildkite_cluster_ids" { + description = "UUIDs of the Buildkite cluster each pipeline belongs to, keyed by pipeline slug" + type = map(string) + default = {} +} + +variable "s3_bucket" { + description = "S3 bucket that release binaries are uploaded to" + type = string + default = "julialangnightlies" +} + +variable "s3_bucket_prefix" { + description = "Prefix under s3_bucket for release binaries" + type = string + default = "bin" +} + +# Bucket + prefix for the scheduled no-GPL builds +# (see pipelines/scheduled/platforms/upload_*.no_gpl.yml) +variable "s3_nogpl_bucket" { + description = "S3 bucket for the scheduled no-GPL builds" + type = string + default = "julialang-nogpl" +} + +variable "s3_nogpl_prefix" { + description = "Prefix under s3_nogpl_bucket" + type = string + default = "bin-nogpl" +} + +# Per-pipeline EPHEMERAL staging buckets (created by this module, with +# lifecycle expiry). The build step of each untrusted pipeline writes its +# unsigned artifacts write-once to ///... +# julia-pr and julia-ci get SEPARATE buckets: the trusted publish pipeline +# only reads the julia-ci bucket, so a pull-request build can never place +# (or, since paths are write-once, pre-claim) anything publish would +# consume. PR consumers (juliaup) read from the julia-pr bucket. +variable "s3_staging_buckets" { + description = "Ephemeral staging bucket per untrusted pipeline, keyed by pipeline slug" + type = map(string) + default = { + "julia-pr" = "julialang-ephemeral-pr" + "julia-ci" = "julialang-ephemeral-ci" + # The julia-buildkite repo's own self-test CI: nothing consumes these + # artifacts (the builds only prove the pipeline code works), and the + # bucket separation means a self-test build can never place anything + # juliaup or julia-publish would read. + "julia-buildkite-ci" = "julialang-ephemeral-buildkite" + } +} + +variable "staging_expiry_days" { + description = "Days before staged objects expire, keyed by pipeline slug" + type = map(number) + default = { + # PR binaries are consumed by juliaup while the PR is open + "julia-pr" = 90 + # publish promotes within hours of staging + "julia-ci" = 30 + # self-test artifacts have no consumers; keep them briefly for humans + "julia-buildkite-ci" = 14 + } +} + +# RSA matching the strength of the previous (pre-KMS) release signing key. +# Must be an RSA spec: the OpenPGP signature assembly in +# utilities/kms_gpg_sign.py only supports RSA. +variable "tarball_key_spec" { + description = "KMS key spec for the GPG release tarball signing key" + type = string + default = "RSA_4096" +} + +# Bearer tokens (codecov, coveralls, buildkite analytics) are inherently +# symmetric secrets, so they live in SSM Parameter Store. The parameter +# *values* are deliberately not managed here (they would end up in the +# Terraform state); store them with ops/23_put_tokens.sh. +variable "ssm_token_prefix" { + description = "SSM Parameter Store prefix for CI telemetry bearer tokens" + type = string + default = "/julia-ci/tokens" +} diff --git a/ops/terraform/versions.tf b/ops/terraform/versions.tf new file mode 100644 index 000000000..caf2fef80 --- /dev/null +++ b/ops/terraform/versions.tf @@ -0,0 +1,24 @@ +terraform { + # aws >= 6.0 is needed for asymmetric (SIGN_VERIFY) key specs on + # aws_kms_external_key (the BYOK notary key). + required_version = ">= 1.5" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + } +} + +provider "aws" { + region = var.aws_region + default_tags { + tags = { + project = "julia-ci" + } + } +} diff --git a/pipelines/main/0_webui.yml b/pipelines/main/0_webui.yml index 8ef688a20..1fc3dffa0 100644 --- a/pipelines/main/0_webui.yml +++ b/pipelines/main/0_webui.yml @@ -1,30 +1,51 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +# This file records the steps configured in the Buildkite webUI; modifying +# this file has no effect -- paste it into the webUI when it changes. +# +# These steps are pasted VERBATIM into BOTH untrusted build pipelines: +# +# julia-pr -- builds pull requests, and nothing else. Settings: +# * Trigger builds after pushing code: OFF +# * Build pull requests: ON (including from third-party +# forked repositories) +# julia-ci -- builds trusted refs only. Settings: +# * Trigger builds after pushing code: ON, +# branch limit: master release-* +# * Build tags: ON (v*) +# * Build pull requests: OFF +# * the nightly build schedule lives here +# +# Keeping the two configurations identical is deliberate: the only +# difference in behavior is downstream, where the rendered julia-publish +# trigger is gated on `pipeline.slug == "julia-ci"`. Neither pipeline +# holds any secrets or signing rights: jobs obtain credentials at runtime +# via Buildkite OIDC (see utilities/aws_oidc.sh), and AWS IAM decides what +# each job identity may do (PR builds: write-once staging uploads only). +# The trusted signing/promotion steps run in julia-publish (see +# pipelines/publish/0_webui.yml), never here. +# +# The same steps are ALSO pasted into a third untrusted pipeline: +# +# julia-buildkite-ci -- self-test CI for the julia-buildkite repository. +# Its pipeline repository is JuliaCI/julia-buildkite (not +# JuliaLang/julia), so its jobs run the repository hook +# .buildkite/hooks/post-checkout, which swaps in a julia +# checkout and pins the external-buildkite meta-data to the +# julia-buildkite commit under test. Settings: +# * Trigger builds after pushing code: ON, branch limit: main +# * Build pull requests: ON (including from third-party +# forked repositories) +# See .buildkite/README.md. steps: - - group: "Upload" + - group: "Launch" steps: - - label: "Unlock secrets, launch pipelines" + - label: "Launch pipelines" agents: queue: "launch" - cryptic_capable: "true" os: "linux" plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/main/launch_unsigned_builders.yml - - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/main/launch_signed_jobs.yml - signature_file: .buildkite/pipelines/main/launch_signed_jobs.yml.signature - command: "true" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/main/launch_untrusted_builders.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/launch_coverage.yml diff --git a/pipelines/main/README.md b/pipelines/main/README.md index 6b9d67bd7..900ca1f32 100644 --- a/pipelines/main/README.md +++ b/pipelines/main/README.md @@ -1,15 +1,22 @@ ## Main pipeline -This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests. +This directory contains most of the builders. They are triggered by GitHub +webhook events (pushes and pull requests). -We have a different main pipeline for each permanent branch. +Builds are split across three Buildkite pipelines by trust level (see +`ops/README.md` for why): -For example: +| Pipeline | Builds | Trust | +| --------------- | ------------------------------------------------- | ------------------------------ | +| `julia-pr` | pull requests | untrusted (stage only) | +| `julia-ci` | `master`, `release-*`, tags, and scheduled nightlies | untrusted to sign; triggers publish | +| `julia-publish` | (triggered by `julia-ci`) signs + promotes | trusted (KMS signing keys) | -| Permanent Branch | Pipeline | -| ---------------- | -------------------------------------------------------------------------------- | -| `master` | [`julia-master`](https://buildkite.com/julialang/julia-master) | -| `release-1.6` | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) | -| `release-1.7` | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) | - -(This is not a complete list.) +Each build step stages its unsigned tarball directly (write-once, no relay +jobs) to a commit-sha-gated path in its pipeline's own ephemeral staging +bucket: `julia-pr` builds go to `julialang-ephemeral-pr` (where juliaup +finds PR binaries) and stop there. Trusted-ref builds run in `julia-ci`, +stage to `julialang-ephemeral-ci`, and trigger `julia-publish`, which +signs and promotes — reading only the `julia-ci` bucket. `julia-publish` +does not build pull requests, so a PR can never reach the signing keys or +feed artifacts into publishing. diff --git a/pipelines/main/launch_coverage.yml b/pipelines/main/launch_coverage.yml new file mode 100644 index 000000000..675e0c8a5 --- /dev/null +++ b/pipelines/main/launch_coverage.yml @@ -0,0 +1,26 @@ +# Launch the coverage pipeline. +# +# Coverage is launched here (a SEPARATE pipeline upload from the build/test +# launchers) so that it is NOT behind the publish barrier: a long scheduled +# coverage run must never delay the julia-publish trigger. The wait+trigger +# live at the end of launch_untrusted_builders.yml instead, inlined with the +# build/test groups they gate. +steps: + - group: "Upload" + steps: + - label: "Launch coverage" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Launch the code coverage job (scheduled julia-ci builds and + # "needs full CI"-labeled PRs; PR builds hold no coverage tokens, + # so they collect coverage without uploading it -- see the + # conditionals inside) + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/coverage/coverage.yml + + agents: + queue: "build" + os: "linux" + arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml deleted file mode 100644 index e3abfaff2..000000000 --- a/pipelines/main/launch_signed_jobs.yml +++ /dev/null @@ -1,42 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: "Launch signed jobs" - command: "true" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - verbose: true - signed_pipelines: - # Launch the upload jobs - - pipeline: .buildkite/pipelines/main/launch_upload_jobs.yml - signature_file: .buildkite/pipelines/main/launch_upload_jobs.yml.signature - inputs: - - .buildkite/pipelines/main/launch_upload_jobs.yml - - .buildkite/utilities/arches_pipeline_upload.sh - - .buildkite/utilities/arches_env.sh - - .buildkite/utilities/build_envs.sh - - .buildkite/utilities/extract_triplet.sh - - .buildkite/utilities/upload_julia.sh - - .buildkite/utilities/aws_config - - .buildkite/utilities/sign_tarball.sh - - .buildkite/utilities/macos/build_dmg.sh - - .buildkite/utilities/macos/codesign.sh - - .buildkite/pipelines/main/platforms/launch_powerpc.jl - - .buildkite/pipelines/main/platforms/upload_linux.arches - - .buildkite/pipelines/main/platforms/upload_linux.yml - - .buildkite/pipelines/main/platforms/upload_macos.arches - - .buildkite/pipelines/main/platforms/upload_macos.yml - - .buildkite/pipelines/main/platforms/upload_windows.arches - - .buildkite/pipelines/main/platforms/upload_windows.yml - - .buildkite/pipelines/main/misc/upload_buildkite_results.yml - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature deleted file mode 100644 index aab63da12..000000000 --- a/pipelines/main/launch_signed_jobs.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__ KX@OxCkd?] H\_#*?3A"pz#F]Eq*=b!9eF @LJkn cN8lN \ No newline at end of file diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml deleted file mode 120000 index 31a037322..000000000 --- a/pipelines/main/launch_unsigned_builders.yml +++ /dev/null @@ -1 +0,0 @@ -launch_unsigned_jobs.yml \ No newline at end of file diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml deleted file mode 100644 index b8ef15cd3..000000000 --- a/pipelines/main/launch_unsigned_jobs.yml +++ /dev/null @@ -1,213 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> launch_unsigned_builders.yml -> misc/llvmpasses.yml -# -# when we could theoretically just have the `webui` launch `misc/llvmpasses.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. - -steps: - - group: "Build" - notify: - - github_commit_status: - context: "Build" - steps: - - label: "Launch build jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 - # A few notes: - # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. - # This is intentional. We want to make sure that this script doesn't suddenly - # break when a new version of the plugin is released. - # 2. We specify the Julia version up to patch, e.g. '1.7.3'. - # This is intentional. We don't do e.g. '1.7', because we want to make sure - # that this script doesn't suddenly break when a new patch release of Julia - # is released. - version: '1.10.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - commands: | - ### Launch Linux build jobs. - # Regular: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Build" \ - ALLOW_FAIL="false" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - ### Launch macOS build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - ### Launch FreeBSD build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_freebsd.arches \ - .buildkite/pipelines/main/platforms/build_freebsd.yml - ### Launch Windows build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_windows.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - group: "Check" - notify: - - github_commit_status: - context: "Check" - steps: - - label: "Launch check jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite - - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - group: "Test" - notify: - - github_commit_status: - context: "Test" - steps: - - label: "Launch test jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - export ALLOW_FAIL="false" - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml - - ### Launch Linux test jobs. - # Regular: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # i686-linux-gnu: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.i686.arches \ - .buildkite/pipelines/main/platforms/test_linux.i686.yml - ### Launch macOS test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml - echo "+++ Finished launching test jobs" - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - group: "Allow Fail" - steps: - - label: "Launch allowed-to-fail build jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail build jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS allowed-to-fail build jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - - label: "Launch allowed-to-fail test jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 - # A few notes: - # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. - # This is intentional. We want to make sure that this script doesn't suddenly - # break when a new version of the plugin is released. - # 2. We specify the Julia version up to patch, e.g. '1.7.3'. - # This is intentional. We don't do e.g. '1.7', because we want to make sure - # that this script doesn't suddenly break when a new patch release of Julia - # is released. - version: '1.10.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - commands: | - export ALLOW_FAIL="true" - - ### Launch Linux allowed-to-fail test jobs. - # Regular: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Allow Fail" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/test_linux.powerpc.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - ### Launch macOS allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD allowed-to-fail jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml - agents: - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/main/launch_untrusted_builders.yml b/pipelines/main/launch_untrusted_builders.yml new file mode 100644 index 000000000..66633640a --- /dev/null +++ b/pipelines/main/launch_untrusted_builders.yml @@ -0,0 +1,69 @@ +# This file launches all the UNTRUSTED build + test jobs -- which in the +# post-cryptic setup is every job of the julia-pr / julia-ci pipelines: +# none of them hold secrets or signing rights (the trusted work happens in +# the separate julia-publish Buildkite pipeline). +# +# LAUNCH-LATENCY OPTIMIZATION: +# +# Previously this file had five separate launch steps (groups Build / Check / +# Test / Allow Fail) that each ran MANY `buildkite-agent pipeline upload` +# calls -- roughly ~30 in total, one per arches-templated platform YAML and +# per static misc YAML. Each upload is a ~1.5s network round-trip, and they +# ran serially, so they dominated CI launch latency. +# +# Now a SINGLE launch step runs `utilities/render_launch_pipeline.py`, which +# renders ONE pre-grouped pipeline document (one `group:` per label: +# Build / Check / Test / Allow Fail / JuliaSyntax / JuliaC, plus the +# wait + julia-publish trigger), and uploads it with ONE `pipeline upload`. +# +# The renderer reproduces the old behaviour EXACTLY: +# * The arches-templated platform YAMLs (platforms/{build,test}_*.yml) are +# rendered once per arch, with the same per-arch variable substitution the +# old per-file `pipeline upload` performed (GROUP / ALLOW_FAIL / arches +# columns). Job grouping, keys, depends_on and soft_fail are preserved. +# * The static / launcher misc YAMLs (analyzegc, gcext, juliac, juliasyntax, +# ...) are emitted VERBATIM -- their double-dollar runtime escapes and their +# launch-env vars (e.g. gcext/test_revise, whose soft_fail is keyed off +# ALLOW_FAIL) are left for THIS single `pipeline upload` to interpolate. +# That is why we still `export ALLOW_FAIL="false"` below. +# * PowerPC is omitted: launch_powerpc.jl only uploaded powerpc arches for +# Julia < 1.12, and is a no-op on current master (1.14). +# +# Yes, this is creating another layer of indirection; the flow now looks like: +# +# [webui] -> launch_untrusted_builders.yml -> render_launch_pipeline.py +# +# however this raises the bar for contributors to add new steps to our CI +# configuration, so I'd rather live with an extra layer of indirection and +# only need to touch the webui configuration when something about the +# launch flow itself changes. + +steps: + - label: ":rocket: Launch jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # gcext.yml / test_revise.yml are emitted verbatim by the renderer and + # still have a soft_fail keyed off ALLOW_FAIL; this single `pipeline + # upload` resolves it -- exactly as the old Test launch step did. + export ALLOW_FAIL="false" + python3 .buildkite/utilities/render_launch_pipeline.py > pipeline.generated.yml + buildkite-agent pipeline upload pipeline.generated.yml + agents: + queue: "launch" + os: "linux" + arch: "x86_64" # prevent from running on PiBots + + # The barrier + trigger of the trusted julia-publish pipeline are emitted by + # render_launch_pipeline.py at the END of the rendered document, in the SAME + # `pipeline upload` as the Build/Check/Test/Allow-Fail groups. That is on + # purpose so the `wait` reliably barriers all of those dynamically-uploaded + # jobs (a `wait` placed in a separately-uploaded file used to race the build + # jobs and fire the trigger before anything was built or staged). + # + # See render_launch_pipeline.py for the wait + the `julia-publish` trigger + # (guarded by `if: pipeline.slug == "julia-ci"`). Coverage is launched by a + # separate upload (launch_coverage.yml) and is intentionally NOT behind this + # barrier. diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml deleted file mode 100644 index 22c1ae236..000000000 --- a/pipelines/main/launch_upload_jobs.yml +++ /dev/null @@ -1,103 +0,0 @@ -# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 - -steps: - - group: "Upload" - steps: - - label: "Launch upload jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 - # A few notes: - # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. - # This is intentional. We want to make sure that this script doesn't suddenly - # break when a new version of the plugin is released. - # 2. We specify the Julia version up to patch, e.g. '1.7.3'. - # This is intentional. We don't do e.g. '1.7', because we want to make sure - # that this script doesn't suddenly break when a new patch release of Julia - # is released. - version: '1.10.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - export ALLOW_FAIL="false" - - ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done - ### Linux: - # Regular Linux upload jobs: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - # PowerPC Linux upload jobs, which we only run for Julia prior to 1.12: - GROUP="Upload" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/upload_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - ### macOS: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_macos.arches \ - .buildkite/pipelines/main/platforms/upload_macos.yml - ### FreeBSD: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_freebsd.arches \ - .buildkite/pipelines/main/platforms/upload_freebsd.yml - ### Windows: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_windows.arches \ - .buildkite/pipelines/main/platforms/upload_windows.yml - - ### Upload test results.json files to buildkite - for OS in linux linux.i686 macos freebsd windows; do \ - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_$${OS}.arches \ - .buildkite/pipelines/main/misc/upload_buildkite_results.yml - done - - # Launch doctest deploy job - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml - - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - - group: "Allow Fail (Upload)" - steps: - - label: "Launch allowed-to-fail upload jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - - export ALLOW_FAIL="true" - - # Launch Linux allowed-to-fail upload jobs - GROUP="Allow Fail (Upload)" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature deleted file mode 100644 index 786b715e6..000000000 --- a/pipelines/main/launch_upload_jobs.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__`3-leyw=lІZ+gsf9Leb"@pv4NSe|NwkQE "XUO; \ No newline at end of file diff --git a/pipelines/main/misc/analyzegc.yml b/pipelines/main/misc/analyzegc.yml index 60d5ae66f..c940d7b27 100644 --- a/pipelines/main/misc/analyzegc.yml +++ b/pipelines/main/misc/analyzegc.yml @@ -11,9 +11,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/llvm_passes.x86_64.tar.gz - rootfs_treehash: "0badf5d8794e21a03ac2ff2d46c1ab538ac02571" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/llvm_passes.x86_64.tar.gz + rootfs_treehash: "630126d5595f428ef824651b9be020ffc485eb6f" workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" diff --git a/pipelines/main/misc/deploy_docs.yml b/pipelines/main/misc/deploy_docs.yml index b0545105e..4466d715a 100644 --- a/pipelines/main/misc/deploy_docs.yml +++ b/pipelines/main/misc/deploy_docs.yml @@ -2,14 +2,16 @@ steps: - group: "Upload" steps: - label: "upload docs" - depends_on: - - "doctest" - # We only upload if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: ((build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/)) || (pipeline.slug == "julia-buildkite") + # The step key IS this job's OIDC identity: julia-oidc-docs-deploy only + # trusts step_key "deploy_docs", from the trusted julia-publish pipeline + # (this file is launched by the publish pipeline, not the build one). + # The key MUST be on this job -- the one that actually assumes the role + # -- not on the launcher step in publish/launch.yml that uploads this + # file (that launcher touches no AWS). There is no `depends_on` doctest + # here: the html docs tarball is read from the commit-sha gated S3 + # staging area (the build pipeline's doctest step stages it), the same + # cross-pipeline handoff used for the binaries. + key: "deploy_docs" # Prevent multiple pipelines from pushing to the `git@github.com:JuliaLang/docs.julialang.org` repo simultaneously concurrency: 1 concurrency_group: 'upload/julialang/upload_docs/docs_julialang_org' @@ -23,23 +25,38 @@ steps: persist_depot_dirs: packages,artifacts,compiled # We maintain multiple Documenter manifests (one manifest for each v1.x minor version of Julia). version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - # Use `aws_uploader` image, since it has things like `git` and `ssh` and whatnot - rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/aws_uploader.x86_64.tar.gz" - rootfs_treehash: "61cb30497604eb1581b3ffd6a8058c150f4cbf8b" + - JuliaCI/sandbox#v2: + # Use the julia_publish image (same trusted toolchain as the + # publish_all step): it has git/ssh/awscli AND the aws-kms-pkcs11 + # PKCS#11 provider used to SSH-sign the docs push with the KMS + # deploy key. (Keep this treehash in sync with publish/launch.yml's + # -- it is the same image.) + rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v8.6/julia_publish.x86_64.tar.gz" + rootfs_treehash: "8b28c5b8937f25827c1ad4b6a3bc1bb90c9e9087" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" - - JuliaCI/cryptic#v2: - files: - - .buildkite/secrets/ssh_docs_deploy - - staticfloat/ssh-agent#v1: - keyfiles: - - .buildkite/secrets/ssh_docs_deploy commands: | - buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" . + echo "--- Download built docs from staging" + # The build pipeline's doctest step stages the html docs under the + # commit-sha-gated key julia-htmldocs.tar.gz (see misc/doctest.yml). + # We fetch it by exact key with a single GetObject -- the staging + # bucket grants public s3:GetObject via its bucket policy, so no IAM + # S3 grant (and no bucket-listing) is needed here. We still assume the + # docs-deploy role below for the KMS-backed SSH signing of the push. + source .buildkite/utilities/aws_oidc.sh docs-deploy + aws s3 cp "s3://${STAGING_BUCKET:-julialang-ephemeral-ci}/${S3_BUCKET_PREFIX:-bin}/${BUILDKITE_COMMIT}/julia-htmldocs.tar.gz" . + + echo "--- Configure SSH signing via AWS KMS" + # The deploy key private half lives in AWS KMS and never leaves + # it: ssh performs each signature through the aws-kms-pkcs11 + # provider, authorized by this job's OIDC identity (the docs-deploy + # role assumed above). No secrets are involved. + mkdir -p ~/.config/aws-kms-pkcs11 + echo '{"slots": [{"kms_key_id": "alias/julia-docs-deploy"}]}' > ~/.config/aws-kms-pkcs11/config.json + export GIT_SSH_COMMAND="ssh -o PKCS11Provider=$${AWS_KMS_PKCS11_PROVIDER:-/usr/lib/x86_64-linux-gnu/pkcs11/aws_kms_pkcs11.so}" echo "--- clone docs.julialang.org" mkdir -p /home/juliaci/.ssh @@ -48,13 +65,13 @@ steps: cd repo echo "--- deploy new docs" - # Delete everything except for `.git` - shopt -s extglob - rm -rf !(".git") - shopt -u extglob + # Delete everything except for `.git` (find, not an extglob `!(.git)` + # glob, which fails to parse unless `shopt -s extglob` is already + # active when the commands block is parsed). + find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + # Extract new docs build - tar -zxf ../julia-*-htmldocs.tar.gz + tar -zxf ../julia-htmldocs.tar.gz # Build commit message echo "build based on $${BUILDKITE_COMMIT}" > ../commit_message @@ -68,19 +85,10 @@ steps: git add . git commit -a -F ../commit_message - if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then - # If we're testing new CI configurations, push to a temporary branch, then delete it - git push origin gh-pages:gh-pages-buildkite-$${BUILDKITE_COMMIT} - git push origin --delete gh-pages-buildkite-$${BUILDKITE_COMMIT} - else - git push origin gh-pages:gh-pages - fi + git push origin gh-pages:gh-pages timeout_in_minutes: 45 agents: - queue: "test" + queue: "default" sandbox_capable: "true" os: "linux" arch: "x86_64" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml index b4e9164a4..dbad83574 100644 --- a/pipelines/main/misc/doctest.yml +++ b/pipelines/main/misc/doctest.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: @@ -41,6 +41,38 @@ steps: echo "--- Compress and upload HTML docs" buildkite-agent artifact upload "$${DOCUMENTER_ARCHIVE}" + + echo "--- Stage HTML docs to S3 for the publish pipeline" + # Mirror the binary handoff (build_julia.sh): a write-once upload to + # this build's commit-sha-gated staging path via the untrusted + # `stage` role, which can do nothing else. The trusted julia-publish + # docs-deploy phase reads it back from the same path (see + # misc/deploy_docs.yml). STAGING_BUCKET selection mirrors + # build_envs.sh; docs are not platform/no-GPL specific, so the prefix + # is the default `bin`. + cd .. + # Documenter only produces the archive when it decides to deploy, + # which it refuses to do for pull-request builds -- on those there + # is nothing to stage (and no publish pipeline consumes it). + if [[ ! -f "doc/$${DOCUMENTER_ARCHIVE}" ]]; then + echo "No $${DOCUMENTER_ARCHIVE} produced (pull-request build); skipping staging." + exit 0 + fi + if [[ "$${BUILDKITE_PIPELINE_SLUG:-}" == "julia-pr" ]]; then + STAGING_BUCKET="julialang-ephemeral-pr" + elif [[ "$${BUILDKITE_PIPELINE_SLUG:-}" == julia-buildkite* ]]; then + STAGING_BUCKET="julialang-ephemeral-buildkite" + else + STAGING_BUCKET="julialang-ephemeral-ci" + fi + source .buildkite/utilities/aws_oidc.sh stage + source .buildkite/utilities/upload_to_s3.sh + # Stage under a stable, version-independent key so the publish + # pipeline's docs-deploy job can fetch it by exact key (a single + # public GetObject -- the staging bucket is public-read) rather than + # listing the bucket, which would need an S3 ListBucket grant. + UPLOAD_TO_S3_ACL=none upload_to_s3 "doc/$${DOCUMENTER_ARCHIVE}" \ + "$${STAGING_BUCKET}/$${S3_BUCKET_PREFIX:-bin}/$${BUILDKITE_COMMIT?}/julia-htmldocs.tar.gz" env: # Give a fake documenter key to get Documenter to try and deploy for us DOCUMENTER_KEY: "" diff --git a/pipelines/main/misc/embedding.yml b/pipelines/main/misc/embedding.yml index 7b9fd7933..d8c850579 100644 --- a/pipelines/main/misc/embedding.yml +++ b/pipelines/main/misc/embedding.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: diff --git a/pipelines/main/misc/gcext.yml b/pipelines/main/misc/gcext.yml index 4e58bd686..aabbe8f46 100644 --- a/pipelines/main/misc/gcext.yml +++ b/pipelines/main/misc/gcext.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: diff --git a/pipelines/main/misc/juliac/test_juliac.yml b/pipelines/main/misc/juliac/test_juliac.yml new file mode 100644 index 000000000..658e13031 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac.yml @@ -0,0 +1,32 @@ +steps: + - group: "JuliaC" + notify: + - github_commit_status: + context: "JuliaC" + steps: + - label: "Launch JuliaC test jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + export ALLOW_FAIL="${ALLOW_FAIL:-false}" + ### Launch Linux JuliaC test jobs: + GROUP="JuliaC" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/juliac/test_juliac_linux.arches \ + .buildkite/pipelines/main/misc/juliac/test_juliac_linux.yml + ### Launch macOS JuliaC test jobs: + GROUP="JuliaC" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/juliac/test_juliac_macos.arches \ + .buildkite/pipelines/main/misc/juliac/test_juliac_macos.yml + ### Launch Windows JuliaC test jobs: + GROUP="JuliaC" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/juliac/test_juliac_windows.arches \ + .buildkite/pipelines/main/misc/juliac/test_juliac_windows.yml + agents: + queue: "launch" + os: "linux" + arch: "x86_64" diff --git a/pipelines/main/misc/juliac/test_juliac_linux.arches b/pipelines/main/misc/juliac/test_juliac_linux.arches new file mode 100644 index 000000000..08ea47320 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_linux.arches @@ -0,0 +1,9 @@ +# ROOTFS_IMAGE_NAME TRIPLET ARCH ARCH_ROOTFS TIMEOUT ROOTFS_TAG ROOTFS_HASH ALLOW_FAIL +package_linux x86_64-linux-gnu x86_64 x86_64 . v7.3 d8de9e0051c5cd69f3dde8ddaefc594d0a71043b . +package_linux aarch64-linux-gnu aarch64 aarch64 . v7.3 d39fb11781df9b03e448ad9318df2d195a3acf26 true + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 +#default ALLOW_FAIL false diff --git a/pipelines/main/misc/juliac/test_juliac_linux.yml b/pipelines/main/misc/juliac/test_juliac_linux.yml new file mode 100644 index 000000000..4a3092e72 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_linux.yml @@ -0,0 +1,43 @@ +steps: + - group: "${GROUP?}" + steps: + - label: ":linux: test JuliaC ${TRIPLET?}" + key: "test-juliac-${TRIPLET?}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + persist_depot_dirs: packages,artifacts,compiled + version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_${TRIPLET}" 'julia-*-linux-*.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-linux-*.tar.gz + ln -s ./usr/bin/julia ./julia + + echo "--- Install and test JuliaC" + unset JULIA_DEPOT_PATH + if [ -f deps/jlutilities/juliac/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.activate("deps/jlutilities/juliac"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="JuliaC", rev="main"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + fi + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + agents: + queue: "test" + sandbox_capable: "true" + os: "linux" + arch: "${ARCH?}" diff --git a/pipelines/main/misc/juliac/test_juliac_macos.arches b/pipelines/main/misc/juliac/test_juliac_macos.arches new file mode 100644 index 000000000..f20eb7875 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_macos.arches @@ -0,0 +1,8 @@ +# OS TRIPLET ARCH TIMEOUT +macos x86_64-apple-darwin x86_64 . +macos aarch64-apple-darwin aarch64 . + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 diff --git a/pipelines/main/misc/juliac/test_juliac_macos.yml b/pipelines/main/misc/juliac/test_juliac_macos.yml new file mode 100644 index 000000000..5b8801625 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_macos.yml @@ -0,0 +1,34 @@ +steps: + - group: "${GROUP?}" + steps: + - label: ":macos: test JuliaC ${TRIPLET?}" + key: "test-juliac-${TRIPLET?}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_${TRIPLET}" 'julia-*-macos-*.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-macos-*.tar.gz + ln -s ./usr/bin/julia ./julia + + # Re-sign after download + .buildkite/utilities/macos/codesign.sh ./usr + + echo "--- Install and test JuliaC" + unset JULIA_DEPOT_PATH + if [ -f deps/jlutilities/juliac/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.activate("deps/jlutilities/juliac"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="JuliaC", rev="main"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + fi + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + agents: + queue: "test" + os: "macos" + arch: "${ARCH}" diff --git a/pipelines/main/misc/juliac/test_juliac_windows.arches b/pipelines/main/misc/juliac/test_juliac_windows.arches new file mode 100644 index 000000000..b56299780 --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_windows.arches @@ -0,0 +1,7 @@ +# OS TRIPLET ARCH TIMEOUT +windows x86_64-w64-mingw32 x86_64 . + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 diff --git a/pipelines/main/misc/juliac/test_juliac_windows.yml b/pipelines/main/misc/juliac/test_juliac_windows.yml new file mode 100644 index 000000000..37980bd5d --- /dev/null +++ b/pipelines/main/misc/juliac/test_juliac_windows.yml @@ -0,0 +1,29 @@ +steps: + - group: "${GROUP?}" + steps: + - label: ":windows: test JuliaC ${TRIPLET?}" + key: "test-juliac-${TRIPLET?}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Download pre-built julia, extract into `usr/` + buildkite-agent artifact download --step "build_${TRIPLET}" 'julia-*-windows-*.tar.gz' . + mkdir -p ./usr + tar -C ./usr --strip-components=1 -zxf julia-*-windows-*.tar.gz + echo "--- Install and test JuliaC" + unset JULIA_DEPOT_PATH + if [ -f deps/jlutilities/juliac/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./usr/bin/julia.exe -e 'using Pkg; Pkg.activate("deps/jlutilities/juliac"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./usr/bin/julia.exe -e 'using Pkg; Pkg.add(name="JuliaC", rev="main"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("JuliaC")' + fi + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + agents: + queue: "test" + os: "windows" + arch: "${ARCH}" diff --git a/pipelines/main/misc/juliasyntax.launch.yml b/pipelines/main/misc/juliasyntax.launch.yml new file mode 100644 index 000000000..c892521e1 --- /dev/null +++ b/pipelines/main/misc/juliasyntax.launch.yml @@ -0,0 +1,31 @@ +# This is its own YAML file so that we can accomplish the following goal: +# If JuliaSyntax has not yet been moved into the JuliaLang/julia repo, +# then PRs to JuliaLang/julia should not display a `JuliaSyntax` PR commit status. + +steps: + - group: "JuliaSyntax" + notify: + - github_commit_status: + context: "JuliaSyntax" + steps: + - label: "Launch JuliaSyntax jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#d8cd48ed4e72ca4ea47a5d02d79fb1bea0ee3e6b: # v1.13.8 + # A few notes: + # 1. We pin the `JuliaCI/julia` plugin to a full commit hash. + # This is intentional. We want to make sure that this script doesn't suddenly + # break when a new version of the plugin is released. + # 2. We specify the Julia version up to patch, e.g. '1.7.3'. + # This is intentional. We don't do e.g. '1.7', because we want to make sure + # that this script doesn't suddenly break when a new patch release of Julia + # is released. + version: '1.10.10' + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/juliasyntax.test.yml + agents: + queue: "launch" + os: "linux" + arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/main/misc/juliasyntax.test.yml b/pipelines/main/misc/juliasyntax.test.yml new file mode 100644 index 000000000..9eb39efc5 --- /dev/null +++ b/pipelines/main/misc/juliasyntax.test.yml @@ -0,0 +1,215 @@ +steps: + - group: "JuliaSyntax" + steps: + # Note: This file only concerns testing JuliaSyntax on older Julia versions + # Testing JuliaSyntax on Julia#master is done separately. + # Specifically, for that case, testing JuliaSyntax is built into the regular `Base.runtests()`, + # so those tests are running in the regular Buildkite platform-specific test jobs. + + # This step has most of the JuliaSyntax test jobs + - label: "juliasyntax {{matrix.agent_os}}/{{matrix.julia_arch}}/{{matrix.julia_version}}" + # We comment out the Buildkite step key for this step, because: + # > You can't use matrix values in other attributes, including step keys and ... + # Source: https://buildkite.com/docs/pipelines/configure/workflows/build-matrix + # key: "juliasyntax-step-key" + timeout_in_minutes: 30 + agents: + queue: "test" + os: "{{matrix.agent_os}}" + arch: "{{matrix.agent_arch}}" + matrix: + setup: + agent_os: + - "linux" + - "windows" + # - "macos" # We handle macOS special, see "adjustments" section below + agent_arch: # arch of the Buildkite agent. Distinct from the Julia binary's arch + - "x86_64" # The "x86_64" agent arch is used for both 32-bit Julia and 64-bit Julia + julia_arch: # arch of the Julia binary to install. Distinct from the agent arch + - "x86_64" # 64-bit Julia + # 32-bit Julia on Linux is handled in a separate step later in this YAML file + # 32-bit Julia on Windows is handled in the "adjustments" section below + julia_version: + # Don't put "1" here. Always include at least major and minor version. + - "1.0" + - "1.6" # old LTS + - "1.10" # current LTS + - "1.12" # latest stable + adjustments: + # macOS x86_64 (Intel): + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.0" + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.6" # old LTS + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.10" # current LTS + - with: + agent_os: "macos" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.12" # latest stable + # macOS aarch64 (Apple Silicon): + # Note: Julia 1.0 and 1.6 did not have native aarch64 binaries for macOS, so we skip those + - with: + agent_os: "macos" + agent_arch: "aarch64" + julia_arch: "aarch64" + julia_version: "1.10" # old LTS + - with: + agent_os: "macos" + agent_arch: "aarch64" + julia_arch: "aarch64" + julia_version: "1.12" # latest stable + # For some reason, the Julia 1.0 tarball for Windows x86_64 doesn't seem to exist. Weird. Skip it for now, try to fix this in the future. + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "x86_64" + julia_version: "1.0" + skip: true + # 32-bit Julia on Windows + # For some reason, the Julia 1.0 tarball for Windows i686 doesn't seem to exist. Weird. Skip it for now, try to fix this in the future. + # - with: + # agent_os: "windows" + # agent_arch: "x86_64" + # julia_arch: "i686" # 32-bit Julia + # julia_version: "1.0" + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "i686" # 32-bit Julia + julia_version: "1.6" # old LTS + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "i686" # 32-bit Julia + julia_version: "1.10" # current LTS + - with: + agent_os: "windows" + agent_arch: "x86_64" + julia_arch: "i686" # 32-bit Julia + julia_version: "1.12" # latest stable + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + isolated_depot: true + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '{{matrix.julia_version}}' + arch: '{{matrix.julia_arch}}' + commands: | + echo "--- Julia versioninfo" + julia -e 'import InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Pkg.dev-ing (by path) the local copy of JuliaSyntax" + export JULIA_PKG_PRECOMPILE_AUTO=0 + # Double dollar signs $$ needed here because of Buildkite stuff + echo "Old JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + # On Windows, the separator is ; + # On other platforms, the separator is : + sep=$$(julia -e 'println(Sys.iswindows() ? ";" : ":")') + # We change the depot path from `depot` to `depot:` + # The trailing colon/semicolon makes sure we can re-use the existing stdlib pkgimages, + # instead of needing to re-precompile them + # https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH + export JULIA_DEPOT_PATH="$${JULIA_DEPOT_PATH:?}$${sep}" + echo "New JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + julia -e 'import Pkg; pspec = Pkg.PackageSpec(path = "./JuliaSyntax"); Pkg.develop(pspec)' + + echo "--- Testing JuliaSyntax on Julia {{matrix.julia_version}}" + unset JULIA_PKG_PRECOMPILE_AUTO + julia -e 'import Pkg; Pkg.test("JuliaSyntax")' + + # This step just has the 32-bit Linux jobs for JuliaSyntax. + # For these jobs, we have to run inside a different rootfs image, because the default 64-bit agent rootfs image will complain + # when we try to run a 32-bit executable. + # The logic for the sandbox plugin is annoying, so I've separated it out. + # AFAICT, there is currently no way to conditionally skip a plugin. + # > Plugins run during the job lifecycle, before the conditional is evaluated + # Source: https://buildkite.com/docs/pipelines/configure/conditionals + - label: "juliasyntax {{matrix.agent_os}}/{{matrix.julia_arch}}/{{matrix.julia_version}} in i686 sandbox" + # We comment out the Buildkite step key for this step, because: + # > You can't use matrix values in other attributes, including step keys and ... + # Source: https://buildkite.com/docs/pipelines/configure/workflows/build-matrix + # key: "juliasyntax-step-key" + timeout_in_minutes: 30 + agents: + queue: "test" + os: "{{matrix.agent_os}}" + arch: "{{matrix.agent_arch}}" + matrix: + setup: + agent_os: + - "linux" + agent_arch: # arch of the Buildkite agent. Distinct from the Julia binary's arch + - "x86_64" # The "x86_64" agent arch is used for both 32-bit Julia and 64-bit Julia + julia_arch: # arch of the Julia binary to install. Distinct from the agent arch + - "i686" # 32-bit Julia + julia_version: + # Don't put "1" here. Always include at least major and minor version. + - "1.0" + - "1.6" # old LTS + - "1.10" # current LTS + - "1.12" # latest stable + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # This is the Julia install *outside* the sandbox. + # We only use this Julia install to run Sandbox.jl + # This is not the Julia install that will be used to run Pkg.test() + isolated_depot: true + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # current LTS. It doesn't really matter what version we pick here. + # arch: '{{matrix.julia_arch}}' # no need to specify the julia_arch for this "outside" Julia. + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.0/agent_linux.i686.tar.gz + rootfs_treehash: "cba648ddb9ac0d292f9c8809652523806674a3d2" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + - JuliaCI/julia#v1: + # This is the Julia install *inside* the sandbox. + # That's the Julia install we actually use for Pkg.test() + isolated_depot: true + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '{{matrix.julia_version}}' + arch: '{{matrix.julia_arch}}' + commands: | + echo "--- Julia versioninfo" + julia -e 'import InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Pkg.dev-ing (by path) the local copy of JuliaSyntax" + export JULIA_PKG_PRECOMPILE_AUTO=0 + # Double dollar signs $$ needed here because of Buildkite stuff + echo "Old JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + # On Windows, the separator is ; + # On other platforms, the separator is : + sep=$$(julia -e 'println(Sys.iswindows() ? ";" : ":")') + # We change the depot path from `depot` to `depot:` + # The trailing colon/semicolon makes sure we can re-use the existing stdlib pkgimages, + # instead of needing to re-precompile them + # https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH + export JULIA_DEPOT_PATH="$${JULIA_DEPOT_PATH:?}$${sep}" + echo "New JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}" + julia -e 'import Pkg; pspec = Pkg.PackageSpec(path = "./JuliaSyntax"); Pkg.develop(pspec)' + + echo "--- Testing JuliaSyntax on Julia {{matrix.julia_version}}" + unset JULIA_PKG_PRECOMPILE_AUTO + julia -e 'import Pkg; Pkg.test("JuliaSyntax")' diff --git a/pipelines/main/misc/llvmpasses.yml b/pipelines/main/misc/llvmpasses.yml index 89876105a..0684e0635 100644 --- a/pipelines/main/misc/llvmpasses.yml +++ b/pipelines/main/misc/llvmpasses.yml @@ -11,9 +11,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: diff --git a/pipelines/main/misc/pdf_docs/build_pdf_docs.yml b/pipelines/main/misc/pdf_docs/build_pdf_docs.yml index 4c0f657b2..17d672186 100644 --- a/pipelines/main/misc/pdf_docs/build_pdf_docs.yml +++ b/pipelines/main/misc/pdf_docs/build_pdf_docs.yml @@ -13,7 +13,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/latex.x86_64.tar.gz rootfs_treehash: "48f7c99f62ac7f2a8212d106059be1dddc3aa6bf" uid: 1000 diff --git a/pipelines/main/misc/sanitizers/asan.yml b/pipelines/main/misc/sanitizers/asan.yml index 9f377ad6a..f3df74443 100644 --- a/pipelines/main/misc/sanitizers/asan.yml +++ b/pipelines/main/misc/sanitizers/asan.yml @@ -11,16 +11,16 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/llvm_passes.x86_64.tar.gz - rootfs_treehash: "0badf5d8794e21a03ac2ff2d46c1ab538ac02571" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/llvm_passes.x86_64.tar.gz + rootfs_treehash: "630126d5595f428ef824651b9be020ffc485eb6f" uid: 1000 gid: 1000 workspaces: - "/cache/repos:/cache/repos" timeout_in_minutes: 180 if: | # We only run the `asan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + (build.branch != "release-1.6") && (build.branch != "release-1.7") commands: | echo "--- Build julia with ASAN" VERBOSE=1 contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} diff --git a/pipelines/main/misc/sanitizers/tsan.yml b/pipelines/main/misc/sanitizers/tsan.yml index 35743227c..e6dbc6b39 100644 --- a/pipelines/main/misc/sanitizers/tsan.yml +++ b/pipelines/main/misc/sanitizers/tsan.yml @@ -8,16 +8,16 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/llvm_passes.x86_64.tar.gz - rootfs_treehash: "0badf5d8794e21a03ac2ff2d46c1ab538ac02571" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/llvm_passes.x86_64.tar.gz + rootfs_treehash: "630126d5595f428ef824651b9be020ffc485eb6f" uid: 1000 gid: 1000 workspaces: - "/cache/repos:/cache/repos" timeout_in_minutes: 120 if: | # We only run the `tsan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + (build.branch != "release-1.6") && (build.branch != "release-1.7") commands: | echo "--- Build julia-debug runtime with TSAN" VERBOSE=1 contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug diff --git a/pipelines/main/misc/test_revise.yml b/pipelines/main/misc/test_revise.yml index 8705ad8da..e511c4036 100644 --- a/pipelines/main/misc/test_revise.yml +++ b/pipelines/main/misc/test_revise.yml @@ -13,9 +13,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: @@ -30,7 +30,11 @@ steps: echo "--- Install and test Revise" unset JULIA_DEPOT_PATH - JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="Revise", rev="master"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("Revise")' + if [ -f deps/jlutilities/revise/Manifest.toml ]; then + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.activate("deps/jlutilities/revise"); Pkg.instantiate(); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("Revise")' + else + JULIA_PKG_PRECOMPILE_AUTO=0 ./julia -e 'using Pkg; Pkg.add(name="Revise", rev="master"); ENV["JULIA_PKG_PRECOMPILE_AUTO"]=1; Pkg.test("Revise")' + fi timeout_in_minutes: 30 soft_fail: ${ALLOW_FAIL?} agents: diff --git a/pipelines/main/misc/trimming.yml b/pipelines/main/misc/trimming.yml index 514beb778..5890ae04a 100644 --- a/pipelines/main/misc/trimming.yml +++ b/pipelines/main/misc/trimming.yml @@ -14,15 +14,15 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" if: | # We only run the `trimming` job on Julia 1.12 and later. - (pipeline.slug != "julia-release-1-dot-10") && (pipeline.slug != "julia-release-1-dot-11") + (build.branch != "release-1.10") && (build.branch != "release-1.11") commands: | # Download pre-built julia, extract into `usr/` buildkite-agent artifact download --step "build_x86_64-linux-gnu" 'julia-*-linux-x86_64.tar.gz' . diff --git a/pipelines/main/misc/upload_buildkite_results.yml b/pipelines/main/misc/upload_buildkite_results.yml deleted file mode 100644 index 0e5bd3759..000000000 --- a/pipelines/main/misc/upload_buildkite_results.yml +++ /dev/null @@ -1,76 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: "upload test results ${TRIPLET?}${USE_RR-}${i686_LABEL-}" - key: "upload_results_${TRIPLET?}${USE_RR-}${i686_LABEL-}" - depends_on: - # Wait for the builder to finish - - step: "build_${TRIPLET?}" - # Wait for the tester to finish - - step: "test_${TRIPLET?}${USE_RR-}${i686_LABEL-}" - allow_failure: true - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - BUILDKITE_ANALYTICS_TOKEN="U2FsdGVkX18BUSpNWNimRUOAh2ku0ugS7x3RRy1CfJBCxTLWArJpa1PXKoILfI7q" - timeout_in_minutes: ${TIMEOUT?} - soft_fail: true - commands: | - echo "--- Downloading test results from test step" - if ! buildkite-agent artifact download --step "test_${TRIPLET?}${USE_RR-}${i686_LABEL-}" "results.tar.gz" .; then - echo "No test results found from test_${TRIPLET?}${USE_RR-}${i686_LABEL-}" - echo "This could mean:" - echo " - The test step hasn't completed successfully yet" - echo " - The test step failed and produced no results" - echo " - This upload job was triggered manually before test completion" - echo "You can re-run this job manually once test results are available." - exit 1 - fi - - # Extract with relaxed permissions handling - if ! tar -xzf results.tar.gz --no-same-owner 2>/dev/null; then - echo "Failed to extract results.tar.gz, trying alternative extraction..." - if ! tar -xzf results.tar.gz --no-same-permissions --no-same-owner 2>/dev/null; then - echo "Could not extract results archive" - exit 1 - fi - fi - - echo "--- Uploading test results and assigning them to the test run" - export JOB_ID="$$(buildkite-agent meta-data get BUILDKITE_TEST_JOB_ID_test_${TRIPLET?}${USE_RR-}${i686_LABEL-} || echo "unknown")" - echo "Test job ID: $$JOB_ID" - - shopt -s nullglob # Ensure empty glob does not return a literal string - for file in results*.json; do - echo "Uploading $$file..." - # We can't use the test-collector plugin because it doesn't provide a way to set a custom `job_id` - curl \ - -X POST \ - --silent \ - --show-error \ - --max-time "${TIMEOUT?}" \ - -H "Authorization: Token token=\"$$BUILDKITE_ANALYTICS_TOKEN\"" \ - -F "data=@\"$$file\"" \ - -F "format=json" \ - -F "run_env[CI]=buildkite" \ - -F "run_env[key]=\"$$BUILDKITE_BUILD_ID\"" \ - -F "run_env[url]=\"$$BUILDKITE_BUILD_URL\"" \ - -F "run_env[branch]=\"$$BUILDKITE_BRANCH\"" \ - -F "run_env[commit_sha]=\"$$BUILDKITE_COMMIT\"" \ - -F "run_env[number]=\"$$BUILDKITE_BUILD_NUMBER\"" \ - -F "run_env[job_id]=\"$$JOB_ID\"" \ - -F "run_env[message]=\"$$BUILDKITE_MESSAGE\"" \ - https://analytics-api.buildkite.com/v1/uploads - echo "" - done - agents: - queue: "test" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - - diff --git a/pipelines/main/misc/whitespace.yml b/pipelines/main/misc/whitespace.yml index 56feb1aea..5c1e1a2d1 100644 --- a/pipelines/main/misc/whitespace.yml +++ b/pipelines/main/misc/whitespace.yml @@ -14,9 +14,9 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" workspaces: - "/cache/repos:/cache/repos" - JuliaCI/julia#v1: diff --git a/pipelines/main/platforms/build_freebsd.yml b/pipelines/main/platforms/build_freebsd.yml index 5ae3a8a33..76c459d04 100644 --- a/pipelines/main/platforms/build_freebsd.yml +++ b/pipelines/main/platforms/build_freebsd.yml @@ -4,6 +4,8 @@ steps: - label: ":freebsd: build ${TRIPLET?}" key: "build_${TRIPLET?}" plugins: + - JuliaCI/coreupload#v2: + core_pattern: "**/*.core" - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" @@ -19,3 +21,4 @@ steps: # Receive some environment mappings from our templating engine TRIPLET: "${TRIPLET?}" MAKE_FLAGS: "${MAKE_FLAGS?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml index 815034b40..9f8f19901 100644 --- a/pipelines/main/platforms/build_linux.yml +++ b/pipelines/main/platforms/build_linux.yml @@ -4,6 +4,10 @@ steps: - label: ":linux: build ${TRIPLET?}" key: "build_${TRIPLET?}" plugins: + - JuliaCI/coreupload#v2: + core_pattern: "**/*.core" + compressor: "zstd" + create_bundle: "true" - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" @@ -11,7 +15,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 @@ -34,3 +38,4 @@ steps: TRIPLET: "${TRIPLET?}" MAKE_FLAGS: "${MAKE_FLAGS?}" ROOTFS_IMAGE_NAME: "${ROOTFS_IMAGE_NAME?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml index 6e9583b34..16d2ce4cd 100644 --- a/pipelines/main/platforms/build_macos.yml +++ b/pipelines/main/platforms/build_macos.yml @@ -4,6 +4,10 @@ steps: - label: ":macos: build ${TRIPLET?}" key: "build_${TRIPLET?}" plugins: + - JuliaCI/coreupload#v2: + core_pattern: "**/*.core" + compressor: "zstd" + create_bundle: "true" - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" @@ -18,3 +22,4 @@ steps: # Receive some environment mappings from our templating engine TRIPLET: "${TRIPLET?}" MAKE_FLAGS: "${MAKE_FLAGS?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_freebsd.yml b/pipelines/main/platforms/test_freebsd.yml index 29b0c40ab..b73fd792f 100644 --- a/pipelines/main/platforms/test_freebsd.yml +++ b/pipelines/main/platforms/test_freebsd.yml @@ -24,3 +24,4 @@ steps: env: JULIA_SHELL: "/usr/local/bin/bash" TRIPLET: "${TRIPLET?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_linux.i686.yml b/pipelines/main/platforms/test_linux.i686.yml index 46b84f624..2cd39ce1b 100644 --- a/pipelines/main/platforms/test_linux.i686.yml +++ b/pipelines/main/platforms/test_linux.i686.yml @@ -20,7 +20,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 @@ -42,3 +42,4 @@ steps: TRIPLET: "${TRIPLET?}" USE_RR: "${USE_RR?}" i686_GROUP: "${i686_GROUP?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml index b69d51c02..5e2a487bc 100644 --- a/pipelines/main/platforms/test_linux.yml +++ b/pipelines/main/platforms/test_linux.yml @@ -20,7 +20,7 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: + - JuliaCI/sandbox#v2: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 @@ -41,3 +41,4 @@ steps: JULIA_SHELL: "/bin/bash" TRIPLET: "${TRIPLET?}" USE_RR: "${USE_RR?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/test_macos.yml b/pipelines/main/platforms/test_macos.yml index 9cb4d8c97..620f4bacc 100644 --- a/pipelines/main/platforms/test_macos.yml +++ b/pipelines/main/platforms/test_macos.yml @@ -25,3 +25,4 @@ steps: env: JULIA_SHELL: "/bin/bash" TRIPLET: "${TRIPLET?}" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/pipelines/main/platforms/upload_freebsd.yml b/pipelines/main/platforms/upload_freebsd.yml deleted file mode 100644 index 082e3bb51..000000000 --- a/pipelines/main/platforms/upload_freebsd.yml +++ /dev/null @@ -1,39 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: ":freebsd: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - files: - - .buildkite/secrets/tarball_signing.gpg - timeout_in_minutes: ${TIMEOUT?} - retry: - automatic: - - exit_status: "*" - limit: 3 - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - os: "freebsd" - arch: "${ARCH}" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml deleted file mode 100644 index abbe59468..000000000 --- a/pipelines/main/platforms/upload_linux.yml +++ /dev/null @@ -1,50 +0,0 @@ -steps: - - group: "${GROUP?}" - steps: - - label: ":linux: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/aws_uploader.x86_64.tar.gz - rootfs_treehash: "61cb30497604eb1581b3ffd6a8058c150f4cbf8b" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - files: - - .buildkite/secrets/tarball_signing.gpg - timeout_in_minutes: ${TIMEOUT?} - soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox_capable: "true" - os: "linux" - arch: "x86_64" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_macos.yml b/pipelines/main/platforms/upload_macos.yml deleted file mode 100644 index c186d27f9..000000000 --- a/pipelines/main/platforms/upload_macos.yml +++ /dev/null @@ -1,42 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: ":macos: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - NOTARIZATION_APPLE_ID="U2FsdGVkX1/b2eyuYolwIaW0DI2YE5tPcMYJ7X7/J5Qz6T/oV6bZEuQqUEet6Jvx" - - NOTARIZATION_APPLE_KEY="U2FsdGVkX1+z0XUSEnj0vLawcT1/U2wx1YDN6s3V9U5+qhbGwRQuXHNJXPmlNe/o" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/macos_codesigning.keychain - timeout_in_minutes: ${TIMEOUT?} - retry: - automatic: - - exit_status: "*" - limit: 3 - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - os: "macos" - arch: "${ARCH}" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_windows.yml b/pipelines/main/platforms/upload_windows.yml deleted file mode 100644 index 67a354b57..000000000 --- a/pipelines/main/platforms/upload_windows.yml +++ /dev/null @@ -1,62 +0,0 @@ -steps: - - group: "Upload" - steps: - - label: ":windows: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Wait for the tester to finish - - "test_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw==" - - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ==" - - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA==" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/windows_codesigning.pfx - - docker#v3.13.0: - # We always use `x86_64` here, as it doesn't matter what the compiler is for uploading - image: "juliapackaging/package-windows-x86_64:${DOCKER_TAG?}" - always-pull: true - command: ["bash", ".buildkite/utilities/upload_julia.sh"] - propagate-environment: true - volumes: - # Mount buildkite-agent as well - - "C:\\buildkite-agent\\bin:C:\\buildkite-agent\\bin" - environment: - # We have to list these here, because buildkite doesn't automatically - # include environment-hook-set variables in a way that the docker - # plugin finds. It's annoying, but at least we have a workaround. - - "JULIA_CPU_THREADS" - # Have to include this for `buildkite-agent` to work: - - "BUILDKITE_AGENT_ACCESS_TOKEN" - # Have to include these for codesigning and uploading - - "AWS_ACCESS_KEY_ID" - - "AWS_SECRET_ACCESS_KEY" - - "AZURE_TENANT_ID" - - "AZURE_CLIENT_ID" - - "AZURE_CLIENT_SECRET" - # These control where we upload, important when running on julia-buildkite - - "S3_BUCKET" - - "S3_BUCKET_PREFIX" - timeout_in_minutes: ${TIMEOUT?} - agents: - queue: "build" - os: "windows" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" diff --git a/pipelines/misc/oidc_smoke_test.yml b/pipelines/misc/oidc_smoke_test.yml new file mode 100644 index 000000000..85b7d64d9 --- /dev/null +++ b/pipelines/misc/oidc_smoke_test.yml @@ -0,0 +1,38 @@ +# OIDC/IAM trust smoke test -- validates role assumption and every trust +# boundary from inside a build pipeline, WITHOUT building julia. +# +# Usage (see also "Validation" in ops/README.md): temporarily replace the +# webUI steps of julia-pr (or julia-ci) with this file's steps and start a +# new build (any recent julia commit); restore the real webUI +# (pipelines/main/0_webui.yml) afterwards. +# +# NOTE: while validating a julia-buildkite revision that julia's +# `.buildkite-external-version` does not point to yet, replace the +# `version:` below with the julia-buildkite commit sha under test. +steps: + - label: ":mag: OIDC trust smoke test" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # The sandbox plugin itself runs on Julia (Sandbox.jl) + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + # The bare agents do not carry the AWS CLI; like the historical + # upload jobs, run inside the `aws_uploader` rootfs, which exists + # to provide aws/git/ssh. + rootfs_url: "https://github.com/JuliaCI/rootfs-images/releases/download/v8.1/aws_uploader.x86_64.tar.gz" + rootfs_treehash: "c031b94d209e3e86507aaa9998c1fa3544f32c72" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + commands: "bash .buildkite/utilities/tests/oidc_smoke_test.sh" + timeout_in_minutes: 10 + agents: + queue: "build" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" diff --git a/pipelines/publish-test/0_webui.yml b/pipelines/publish-test/0_webui.yml new file mode 100644 index 000000000..c139090fd --- /dev/null +++ b/pipelines/publish-test/0_webui.yml @@ -0,0 +1,29 @@ +# Steps configured in the `julia-publish-test-nosecrets` pipeline's webUI; +# modifying this file has no effect -- paste it into the webUI when it changes. +# +# This is the ISOLATED, NON-PRODUCTION debug pipeline. It runs the real publish +# flow (download -> KMS-sign -> promote) but against THROWAWAY test KMS keys +# (self-signed certs) and the `julialang-test-publish` bucket -- so it can be +# triggered freely to debug the publish flow without touching any production +# key, bucket, or the public `julia-latest-*` pointers. +# +# Unlike `julia-publish`, this pipeline is NOT a trust boundary: it can only +# assume `julia-oidc-publish-test`, which is scoped to the test keys + test +# bucket. So it does NOT need pull-request builds disabled or branch limiting. +# It is triggered manually (never cross-triggered from julia-ci). +# +# Input is seeded with ops/seed_test_staging.sh; trigger a build on the same +# (real master) commit so verify_trusted_commit.sh passes. +steps: + - group: "Publish (test)" + steps: + - label: "Launch publish-test jobs" + agents: + queue: "default" + os: "linux" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/publish-test/launch.yml diff --git a/pipelines/publish-test/launch.yml b/pipelines/publish-test/launch.yml new file mode 100644 index 000000000..76a0b71d0 --- /dev/null +++ b/pipelines/publish-test/launch.yml @@ -0,0 +1,92 @@ +# Launched by the julia-publish-test-nosecrets pipeline. Runs the exact same +# publish.sh flow as production, but the env below redirects every signing key +# to a THROWAWAY *-test KMS key (self-signed cert) and every bucket to the +# isolated julialang-test-publish bucket, and skips the two signers that have +# no non-production equivalent (Windows Azure Trusted Signing, macOS Apple +# notarization). Nothing here can touch a production key, bucket, or pointer. +# +# See ops/terraform/test_publish.tf (test keys + bucket + julia-oidc-publish-test +# role) and ops/seed_test_staging.sh (seed the staged input). +steps: + - group: "Publish (test)" + steps: + - label: ":test_tube: publish all (test)" + key: "publish_all" + concurrency: 1 + concurrency_group: 'publish-test/julialangnightlies' + concurrency_method: eager + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # OUTER julia: runs on the bare agent, only to launch Sandbox.jl + # (the sandbox plugin requires a host julia). NOT visible inside + # the sandbox -- plugins before sandbox#v2 run outside it. + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + # Same toolchain image as the production publish step. + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.6/julia_publish.x86_64.tar.gz + rootfs_treehash: "8b28c5b8937f25827c1ad4b6a3bc1bb90c9e9087" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + # Mount the julia plugin's install + depot so the INNER julia + # plugin below reuses the already-downloaded julia instead of + # re-fetching it inside the sandbox. + - "/cache/julia-buildkite-plugin:/cache/julia-buildkite-plugin" + - JuliaCI/julia#v1: + # INNER julia: plugins after sandbox#v2 run *inside* the sandbox, + # so this puts `julia` on PATH within the rootfs. upload_julia.sh + # uses it to patch pkgimage checksums in the cross-built (foreign + # platform) trees -- the published binaries can't run on this agent. + version: '1.10' # GREP_ME: Keep in sync with the outer julia version. + timeout_in_minutes: 180 + commands: "bash .buildkite/utilities/publish.sh" + agents: + queue: "default" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" + env: + # Scope the test to the Linux + macOS triplets. Linux exercises the + # GPG-sign path (content-agnostic). macOS unpacks the staged unsigned + # Julia.app (assembled by the build_ step), rcodesign-codesigns its + # mach-o with the test KMS key + self-signed cert, runs the cross-mode + # pkgimage checksum patch, and builds the .dmg (notarization skipped). + PUBLISH_ARCHES_FILES: ".buildkite/pipelines/main/platforms/upload_linux.arches .buildkite/pipelines/main/platforms/upload_macos.arches" + # Assume the throwaway test role (test keys + test bucket only). + PUBLISH_OIDC_MODE: "publish-test" + # Read staged input from the julia-ci ephemeral bucket (where a real + # build stages its unsigned tarballs/.app -- the bucket is public-read, + # so the test role can GetObject), but PROMOTE only to the throwaway + # test bucket. So this consumes real staged artifacts and exercises the + # full sign+promote without touching any production release path. + STAGING_BUCKET: "julialang-ephemeral-ci" + S3_BUCKET: "julialang-test-publish" + S3_BUCKET_PREFIX: "bin" + # The test bucket disables object ACLs (BucketOwnerEnforced, public + # read via bucket policy), so the promote must not send --acl. + UPLOAD_TO_S3_ACL: "none" + # Allow re-publishing the same commit repeatedly ("test to our heart's + # content"): overwrite unconditionally instead of write-once. Safe only + # because the throwaway test role permits unconditional PutObject; the + # production publish role's IAM denies it regardless of this flag. + UPLOAD_TO_S3_OVERWRITE: "1" + # Throwaway signing material (test KMS keys + self-signed cert/pubkey). + MACOS_CODESIGN_KMS_KEY: "alias/julia-macos-codesigning-test" + MACOS_CODESIGN_CERT: ".buildkite/utilities/macos/developer_id_test.pem" + TARBALL_SIGNING_KMS_KEY: "alias/julia-tarball-signing-test" + # Derive the GPG pubkey identity from KMS at runtime (GetPublicKey + + # DescribeKey for the creation date) instead of committing a throwaway + # test pubkey. Empty PUBKEY selects derive mode. + TARBALL_SIGNING_PUBKEY: "" + # The two signers with no non-production equivalent are skipped: + # Windows Authenticode (Azure Trusted Signing) and macOS notarization + # (Apple App Store Connect). The .dmg is still built and codesigned. + PUBLISH_SKIP_WINDOWS_SIGN: "1" + PUBLISH_SKIP_NOTARIZATION: "1" + # Inno Setup prefix baked into the rootfs (as in production publish). + WINEPREFIX: "/home/juliaci/.wine" diff --git a/pipelines/publish/0_webui.yml b/pipelines/publish/0_webui.yml new file mode 100644 index 000000000..09bc77328 --- /dev/null +++ b/pipelines/publish/0_webui.yml @@ -0,0 +1,33 @@ +# This file records the steps configured in the julia-publish pipeline's +# webUI; modifying this file has no effect -- paste it into the webUI when +# it changes. +# +# SECURITY: the julia-publish pipeline is the trust boundary. It MUST be +# configured in Buildkite with: +# * Trigger builds after pushing code: OFF +# * Build pull requests: OFF (including from third-party forked +# repositories) +# * Branch limiting: master release-* (and build tags v*) +# Builds are created only by the `trigger` step at the end of julia-ci +# (the inlined wait+trigger at the end of pipelines/main/launch_untrusted_builders.yml). +# Because a pull request can never produce a build under this pipeline slug, +# the slug itself is what the julia-oidc-publish / julia-oidc-docs-deploy IAM +# roles trust -- not the (PR-spoofable) branch name. The publish jobs also +# run verify_trusted_commit.sh as a backstop. +# +# This webUI step loads its launch steps from a pinned julia-buildkite (via +# the external-buildkite plugin), NOT from the triggered build's tree, so a +# malicious build commit cannot alter the publish steps. +steps: + - group: "Publish" + steps: + - label: "Launch publish jobs" + agents: + queue: "default" + os: "linux" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/publish/launch.yml diff --git a/pipelines/publish/launch.yml b/pipelines/publish/launch.yml new file mode 100644 index 000000000..132952544 --- /dev/null +++ b/pipelines/publish/launch.yml @@ -0,0 +1,90 @@ +# Launched by the julia-publish pipeline. A single trusted step signs and +# promotes every staged build to its final release location, then deploys +# docs. This is the only place release secrets (KMS signing keys) are used. +steps: + - group: "Publish" + steps: + - label: ":lock: publish all" + key: "publish_all" + # No in-pipeline build dependency: reads the unsigned tarballs from + # the commit-sha-gated staging area in S3 (written by the build + # pipeline). One job, all triplets. + concurrency: 1 + concurrency_group: 'publish/julialangnightlies' + concurrency_method: eager + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # OUTER julia: runs on the bare agent, only to launch Sandbox.jl + # (the sandbox plugin requires a host julia). Plugins *before* + # sandbox#v2 run outside the sandbox, so this one is NOT visible + # inside it -- the inner julia#v1 below provides the in-sandbox julia. + persist_depot_dirs: packages,artifacts,compiled + version: '1.10' # GREP_ME: Keep this updated to LTS. Do not set it to '1' or 'stable' or 'latest'. + - JuliaCI/sandbox#v2: + # Run inside the julia_publish rootfs, which carries the full + # signing/packaging toolchain (aws CLI, p7zip, libdmg-hfsplus + + # newfs_hfs, wine + Inno Setup, jsign, python). Without this the + # step runs on a bare agent and fails with `aws: command not + # found`. x86_64-only image; built by JuliaCI/rootfs-images. + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.6/julia_publish.x86_64.tar.gz + rootfs_treehash: "8b28c5b8937f25827c1ad4b6a3bc1bb90c9e9087" + uid: 1000 + gid: 1000 + workspaces: + # Mount the git object cache so upload_julia.sh's version + # introspection (git describe) resolves cleanly inside the + # sandbox, same as the build_linux step. + - "/cache/repos:/cache/repos" + # Mount the julia plugin's install + depot so the inner julia + # plugin below reuses the already-downloaded julia instead of + # re-fetching it inside the sandbox. + - "/cache/julia-buildkite-plugin:/cache/julia-buildkite-plugin" + - JuliaCI/julia#v1: + # INNER julia: plugins after sandbox#v2 run *inside* the sandbox, + # so this puts `julia` on PATH within the rootfs. upload_julia.sh + # uses it to patch the pkgimage checksums inside the (foreign- + # platform) signed install trees -- the published binaries cannot + # run on this agent. + version: '1.10' # GREP_ME: Keep in sync with the outer julia version. + timeout_in_minutes: 180 + commands: "bash .buildkite/utilities/publish.sh" + agents: + # The signing/packaging toolchain is provided by the julia_publish + # rootfs via the sandbox plugin above (rcodesign is fetched at + # runtime). The agent only needs to be able to nest a Sandbox.jl + # sandbox, hence sandbox_capable. + queue: "default" + sandbox_capable: "true" + os: "linux" + arch: "x86_64" + env: + # The Inno Setup 6 prefix is baked into the julia_publish rootfs at + # /home/juliaci/.wine (owned by uid 1000). upload_julia.sh runs + # `wine ISCC.exe` without setting WINEPREFIX, so it would otherwise + # default to $HOME/.wine -- pin it here so the baked prefix is used + # regardless of what HOME the sandbox sets. + WINEPREFIX: "/home/juliaci/.wine" + # Identify the Azure Trusted Signing app registration for Windows + # Authenticode signing (not secrets; auth is via Buildkite OIDC + # workload identity federation -- see windows/codesign.sh and + # ops/terraform/azure). + AZURE_TENANT_ID: "8d3de454-9fdc-4091-90af-9cf79dcabe79" + AZURE_CLIENT_ID: "a1fa180a-5312-4673-a98c-7cde383bae9c" + + # Launcher only: uploads the docs-deploy pipeline and touches no AWS, so + # it must NOT carry key "deploy_docs" -- that key belongs to the actual + # role-assuming "upload docs" job (see misc/deploy_docs.yml), and a + # duplicate key across the build is rejected. + - label: "deploy docs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: "buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml" + agents: + queue: "default" + os: "linux" + arch: "x86_64" diff --git a/pipelines/scheduled/0_webui.yml b/pipelines/scheduled/0_webui.yml index f17513039..4d0087321 100644 --- a/pipelines/scheduled/0_webui.yml +++ b/pipelines/scheduled/0_webui.yml @@ -1,30 +1,23 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +# This file records the steps configured in the Buildkite webUI; modifying +# this file has no effect -- paste it into the webUI when it changes. +# +# There is no secrets management here anymore: jobs that need credentials +# obtain them at runtime via Buildkite OIDC (see utilities/aws_oidc.sh), +# and AWS IAM decides what each job identity may do. The signed/unsigned +# split between pipeline FILES is therefore gone; the trust split lives at +# the level of Buildkite PIPELINES instead (untrusted julia-pr / julia-ci +# vs the trusted julia-publish -- see ops/README.md). steps: - group: "Launch scheduled jobs" steps: - - label: "Unlock secrets, launch pipelines" + - label: "Launch pipelines" agents: - queue: "launch" - cryptic_capable: "true" + queue: "build" os: "linux" plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml - - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/scheduled/launch_signed_jobs.yml - signature_file: .buildkite/pipelines/scheduled/launch_signed_jobs.yml.signature - command: "true" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/launch_untrusted_builders.yml + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/launch_coverage.yml diff --git a/pipelines/scheduled/coverage/coverage.yml b/pipelines/scheduled/coverage/coverage.yml index 132293046..0dbb93d69 100644 --- a/pipelines/scheduled/coverage/coverage.yml +++ b/pipelines/scheduled/coverage/coverage.yml @@ -1,9 +1,16 @@ steps: - group: "Coverage" + # Scheduled (julia-ci) builds, plus "needs full CI"-labeled PRs. Only + # julia-ci holds the codecov / coveralls bearer tokens; PR builds run + # the same coverage collection but skip the uploads (the lcov tarball + # is still available as a buildkite artifact). + if: (build.source == "schedule") || (build.pull_request.labels includes "needs full CI") + notify: + - github_commit_status: + context: "Coverage" steps: - label: ":linux: coverage (x86_64)" key: "coverage-linux" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -12,17 +19,27 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/package_linux.x86_64.tar.gz - rootfs_treehash: "4dcde853eb5baaa0a8f087b633eaf955dc94b5dc" + - JuliaCI/sandbox#v2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.5/package_linux.x86_64.tar.gz + rootfs_treehash: "8b46765146862479f0a8946f2ef783a8bf93b20f" uid: 1000 gid: 1000 - - JuliaCI/cryptic#v2: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - timeout_in_minutes: 360 # 6h + timeout_in_minutes: 600 # 10h commands: | + echo "--- Obtain coverage tokens" + # Coverage tokens are fetched from SSM Parameter Store with this + # job's OIDC identity; no static secrets exist in CI config. + # Only julia-ci holds tokens (PR builds and the julia-buildkite + # self-test have no tokens role): collect coverage anyway, skip + # the uploads below. + if [[ "$${BUILDKITE_PIPELINE_SLUG}" != "julia-ci" ]]; then + echo "No coverage tokens in this pipeline; coverage uploads will be skipped" + else + source .buildkite/utilities/aws_oidc.sh tokens + export CODECOV_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/codecov_token --query Parameter.Value --output text)" + export COVERALLS_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text)" + fi + echo "--- Build Julia from source" make --output-sync -j$${JULIA_NUM_THREADS} @@ -49,7 +66,11 @@ steps: buildkite-agent artifact upload lcov_files.tar.gz echo "--- Process and upload coverage information" - ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + if [[ -n "$${CODECOV_TOKEN:-}" ]]; then + ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + else + echo "No coverage tokens; skipping upload" + fi agents: queue: "test" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -57,14 +78,11 @@ steps: os: "linux" arch: "x86_64" env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} - label: ":macos: coverage (aarch64)" key: "coverage-macos-aarch64" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -73,12 +91,22 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - JuliaCI/cryptic#v2: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" timeout_in_minutes: 300 # 5h commands: | + echo "--- Obtain coverage tokens" + # Coverage tokens are fetched from SSM Parameter Store with this + # job's OIDC identity; no static secrets exist in CI config. + # Only julia-ci holds tokens (PR builds and the julia-buildkite + # self-test have no tokens role): collect coverage anyway, skip + # the uploads below. + if [[ "$${BUILDKITE_PIPELINE_SLUG}" != "julia-ci" ]]; then + echo "No coverage tokens in this pipeline; coverage uploads will be skipped" + else + source .buildkite/utilities/aws_oidc.sh tokens + export CODECOV_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/codecov_token --query Parameter.Value --output text)" + export COVERALLS_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text)" + fi + echo "--- Build Julia from source" make -j$${JULIA_NUM_THREADS} @@ -105,21 +133,22 @@ steps: buildkite-agent artifact upload lcov_files.tar.gz echo "--- Process and upload coverage information" - ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + if [[ -n "$${CODECOV_TOKEN:-}" ]]; then + ./julia --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + else + echo "No coverage tokens; skipping upload" + fi agents: queue: "test" os: "macos" arch: "aarch64" env: JULIA_SHELL: "/bin/bash" - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} - label: ":windows: coverage (x86_64)" key: "coverage-windows-x86_64" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -128,14 +157,18 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - JuliaCI/cryptic#v2: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - docker#v3.13.0: - image: "juliapackaging/package-windows-x86_64:v7.10" + image: "juliapackaging/package-windows-x86_64:v8.5" always-pull: true command: ["bash", "-c", " + echo '--- Obtain coverage tokens' && + if [[ $${BUILDKITE_PIPELINE_SLUG} == julia-pr ]]; then + echo 'PR build: no coverage tokens; coverage uploads will be skipped'; + else + source .buildkite/utilities/aws_oidc.sh tokens && + export CODECOV_TOKEN=$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/codecov_token --query Parameter.Value --output text) && + export COVERALLS_TOKEN=$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text); + fi && echo '--- Build Julia from source' && make -j$${JULIA_NUM_THREADS} VERBOSE=1 && echo '--- Print Julia version info' && @@ -156,7 +189,11 @@ steps: tar -zcf lcov_files.tar.gz lcov_files/ && buildkite-agent artifact upload lcov_files.tar.gz && echo '--- Process and upload coverage information' && - ./usr/bin/julia.exe --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + if [[ -n $${CODECOV_TOKEN:-} ]]; then + ./usr/bin/julia.exe --color=yes .buildkite/pipelines/scheduled/coverage/upload_coverage.jl; + else + echo 'No coverage tokens (PR build); skipping upload'; + fi "] propagate-environment: true volumes: @@ -174,9 +211,8 @@ steps: - "BUILDKITE_BUILD_AUTHOR_EMAIL" - "BUILDKITE_BUILD_CREATOR_EMAIL" - "BUILDKITE_PULL_REQUEST" - - "CODECOV_TOKEN" - - "COVERALLS_TOKEN" - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_PIPELINE_SLUG" timeout_in_minutes: 540 # 9h agents: queue: "test" @@ -184,14 +220,11 @@ steps: arch: "x86_64" env: JULIA_SHELL: "/bin/bash" - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} - label: "Finalize parallel coverage" key: "coverage-finalize" - if: (pipeline.slug == "julia-buildkite-scheduled") || (pipeline.slug == "julia-master-scheduled") || (build.pull_request.labels includes "needs full CI") depends_on: - step: "coverage-linux" allow_failure: true @@ -207,19 +240,24 @@ steps: # Drop default "registries" directory, so it is not persisted from execution to execution persist_depot_dirs: packages,artifacts,compiled version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - JuliaCI/cryptic#v2: - variables: - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" timeout_in_minutes: 10 commands: | echo "--- Signal completion of parallel coverage uploads" - julia --color=yes .buildkite/pipelines/scheduled/coverage/finish_parallel_coverage.jl + # PR builds hold no tokens and did not upload anything, so there + # is nothing to finalize there. + if [[ "$${BUILDKITE_PIPELINE_SLUG}" == "julia-pr" ]]; then + echo "PR build: nothing was uploaded; nothing to finalize" + else + # The coveralls token is fetched from SSM Parameter Store with + # this job's OIDC identity; no static secrets exist in CI config. + source .buildkite/utilities/aws_oidc.sh tokens + export COVERALLS_TOKEN="$$(aws ssm get-parameter --with-decryption --name /julia-ci/tokens/coveralls_token --query Parameter.Value --output text)" + julia --color=yes .buildkite/pipelines/scheduled/coverage/finish_parallel_coverage.jl + fi agents: queue: "test" os: "linux" arch: "x86_64" env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} # Set build number for Coveralls parallel mode COVERALLS_SERVICE_NUMBER: ${BUILDKITE_BUILD_NUMBER} diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature deleted file mode 100644 index 71698ff81..000000000 --- a/pipelines/scheduled/coverage/coverage.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__aMmratO(\SjAgB@QYJ7Ab)qRkjq{w]$"B֧̈́ \ No newline at end of file diff --git a/pipelines/scheduled/coverage/upload_coverage.jl b/pipelines/scheduled/coverage/upload_coverage.jl index 08b90360b..138047188 100644 --- a/pipelines/scheduled/coverage/upload_coverage.jl +++ b/pipelines/scheduled/coverage/upload_coverage.jl @@ -4,6 +4,7 @@ import Pkg import Logging import TOML +Pkg.develop(path="./JuliaSyntax") Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1") import Coverage diff --git a/pipelines/scheduled/launch_coverage.yml b/pipelines/scheduled/launch_coverage.yml new file mode 100644 index 000000000..eb2786f89 --- /dev/null +++ b/pipelines/scheduled/launch_coverage.yml @@ -0,0 +1,28 @@ +# Launch the scheduled coverage pipeline. +# +# Historically this launched "signed pipelines" (treehash-verified by the +# cryptic plugin) because they carried decryption rights for static +# secrets. There are no pipeline-held secrets anymore: jobs obtain +# credentials at runtime via Buildkite OIDC, and AWS IAM decides what each +# job identity (pipeline/ref/step) may do. +# +# The no-GPL upload relay jobs are gone too: the no-GPL build steps stage +# their unsigned tarballs directly to the julia-ci ephemeral staging bucket +# (under the bin-nogpl prefix; see build_envs.sh). Signing + promotion to +# the julialang-nogpl bucket belongs in the julia-publish pipeline (TODO: +# wire the no-GPL triplets into utilities/publish.sh). +steps: + - group: "Launch scheduled jobs" + steps: + - label: "Launch upload jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Launch the code coverage job + buildkite-agent pipeline upload .buildkite/pipelines/scheduled/coverage/coverage.yml + agents: + queue: "build" + os: "linux" + arch: "x86_64" # prevent from running on PiBots diff --git a/pipelines/scheduled/launch_signed_jobs.yml b/pipelines/scheduled/launch_signed_jobs.yml deleted file mode 100644 index 88e87863a..000000000 --- a/pipelines/scheduled/launch_signed_jobs.yml +++ /dev/null @@ -1,47 +0,0 @@ -steps: - - group: "Launch scheduled jobs" - steps: - - label: "Launch signed jobs" - command: "true" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - verbose: true - signed_pipelines: - # Launch the code coverage job - - pipeline: .buildkite/pipelines/scheduled/coverage/coverage.yml - signature_file: .buildkite/pipelines/scheduled/coverage/coverage.yml.signature - inputs: - - .buildkite/pipelines/scheduled/coverage/coverage.yml - - .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl - - .buildkite/pipelines/scheduled/coverage/upload_coverage.jl - - .buildkite/pipelines/scheduled/coverage/finish_coveralls_parallel.jl - # Launch the upload jobs - - pipeline: .buildkite/pipelines/scheduled/launch_upload_jobs.yml - signature_file: .buildkite/pipelines/scheduled/launch_upload_jobs.yml.signature - inputs: - - .buildkite/pipelines/main/launch_upload_jobs.yml - - .buildkite/utilities/arches_pipeline_upload.sh - - .buildkite/utilities/arches_env.sh - - .buildkite/utilities/build_envs.sh - - .buildkite/utilities/extract_triplet.sh - - .buildkite/utilities/upload_julia.sh - - .buildkite/utilities/sign_tarball.sh - - .buildkite/utilities/macos/build_dmg.sh - - .buildkite/utilities/macos/codesign.sh - - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml - - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches - - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml - - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches - - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml - - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/scheduled/launch_signed_jobs.yml.signature b/pipelines/scheduled/launch_signed_jobs.yml.signature deleted file mode 100644 index f6eacf665..000000000 --- a/pipelines/scheduled/launch_signed_jobs.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__UXzz!Fh@DAP&)$%2ޱß0)XkN},F04ER^jLL}C|]I \ No newline at end of file diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_untrusted_builders.yml similarity index 91% rename from pipelines/scheduled/launch_unsigned_jobs.yml rename to pipelines/scheduled/launch_untrusted_builders.yml index b7bd5566f..e8084bab6 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_untrusted_builders.yml @@ -1,10 +1,10 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. +# This file launches all the UNTRUSTED build + test jobs; no job in the +# build pipelines holds secrets or signing rights (the trusted work happens +# in the separate julia-publish Buildkite pipeline). # # Yes, this is creating another layer of indirection; the flow now looks like: # -# [webui] -> launch_unsigned_builders.yml -> misc/llvmpasses.yml +# [webui] -> launch_untrusted_builders.yml -> misc/llvmpasses.yml # # when we could theoretically just have the `webui` launch `misc/llvmpasses.yml`, # however this raises the bar for contributors to add new (unsigned) steps to diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml deleted file mode 100644 index f8016246d..000000000 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 - -steps: - - group: "Upload (no GPL)" - steps: - - label: "Launch upload jobs (no GPL)" - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - - # Launch `upload_*` jobs to store tarballs into S3 once tests are done - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml - - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - cryptic_capable: "true" - queue: "launch" - os: "linux" - arch: "x86_64" # prevent from running on PiBots - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/scheduled/launch_upload_jobs.yml.signature b/pipelines/scheduled/launch_upload_jobs.yml.signature deleted file mode 100644 index 20a19ebacad8bd8ca34d6dd674cff1ed04b57785..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmV-m0H6O;VQh3|WM5zGo;4|zW@DU7sMQC%&I)}hhS8H6tLqt@00OQEITxrMc57NL z-z1)>37Qrcos0Q_)d|Efl4(pZ)w6DsMkVArrn&9>+IpoQ^&5}%kOSDJ`zXCB C|1Tu~ diff --git a/pipelines/scheduled/platforms/upload_linux.no_gpl.yml b/pipelines/scheduled/platforms/upload_linux.no_gpl.yml deleted file mode 100644 index 099aeee96..000000000 --- a/pipelines/scheduled/platforms/upload_linux.no_gpl.yml +++ /dev/null @@ -1,56 +0,0 @@ -steps: - - group: "Upload (no GPL)" - steps: - - label: ":linux: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - # We only upload to S3 if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite") || (pipeline.slug == "julia-buildkite-scheduled") || (build.pull_request.labels includes "needs full CI") - depends_on: - # Wait for the build to finish - - "build_${TRIPLET?}" - # There are no tests to wait on - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.11' # GREP_ME: Keep this updated. Do not set it to '1' or 'stable' or 'latest'. - - staticfloat/sandbox#v2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v6.00/aws_uploader.x86_64.tar.gz - rootfs_treehash: "61cb30497604eb1581b3ffd6a8058c150f4cbf8b" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - files: - - .buildkite/secrets/tarball_signing.gpg - timeout_in_minutes: ${TIMEOUT?} - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox_capable: "true" - os: "linux" - arch: "x86_64" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - S3_BUCKET: "julialang-nogpl" - S3_BUCKET_PREFIX: "bin-nogpl" diff --git a/pipelines/scheduled/platforms/upload_macos.no_gpl.yml b/pipelines/scheduled/platforms/upload_macos.no_gpl.yml deleted file mode 100644 index e3c40cd18..000000000 --- a/pipelines/scheduled/platforms/upload_macos.no_gpl.yml +++ /dev/null @@ -1,45 +0,0 @@ -steps: - - group: "Upload (no GPL)" - steps: - - label: ":macos: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - # We only upload to S3 if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite") || (pipeline.slug == "julia-buildkite-scheduled") || (build.pull_request.labels includes "needs full CI") - depends_on: - # Wait for the build to finish - - "build_${TRIPLET?}" - # There are no tests to wait on - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - NOTARIZATION_APPLE_ID="U2FsdGVkX1/b2eyuYolwIaW0DI2YE5tPcMYJ7X7/J5Qz6T/oV6bZEuQqUEet6Jvx" - - NOTARIZATION_APPLE_KEY="U2FsdGVkX1+z0XUSEnj0vLawcT1/U2wx1YDN6s3V9U5+qhbGwRQuXHNJXPmlNe/o" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/macos_codesigning.keychain - timeout_in_minutes: ${TIMEOUT?} - commands: "bash .buildkite/utilities/upload_julia.sh" - agents: - queue: "build" - os: "macos" - arch: "${ARCH}" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - S3_BUCKET: "julialang-nogpl" - S3_BUCKET_PREFIX: "bin-nogpl" diff --git a/pipelines/scheduled/platforms/upload_windows.no_gpl.yml b/pipelines/scheduled/platforms/upload_windows.no_gpl.yml deleted file mode 100644 index c93485cfe..000000000 --- a/pipelines/scheduled/platforms/upload_windows.no_gpl.yml +++ /dev/null @@ -1,69 +0,0 @@ -steps: - - group: "Upload (no GPL)" - steps: - - label: ":windows: upload ${TRIPLET?}" - key: "upload_${TRIPLET?}" - # We only upload to S3 if one of the following criteria are true: - # 1. The branch is `master`. - # 2. The branch is `release-*`. - # 3. The build is a tag build AND the tag is `v*`. - # 4. The pipeline is `julia-buildkite`. - if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite-scheduled") || (build.pull_request.labels includes "needs full CI") - depends_on: - # Wait for the builder to finish - - "build_${TRIPLET?}" - # Prevent multiple pipelines from uploading to S3 simultaneously - # It is okay for two different triplets to upload simultaneously - concurrency: 1 - concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' - concurrency_method: eager - plugins: - - JuliaCI/external-buildkite#v1: - version: "./.buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/cryptic#v2: - variables: - - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" - - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw==" - - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ==" - - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA==" - files: - - .buildkite/secrets/tarball_signing.gpg - - .buildkite/secrets/windows_codesigning.pfx - - docker#v3.13.0: - # We always use `x86_64` here, as it doesn't matter what the compiler is for uploading - image: "juliapackaging/package-windows-x86_64:${DOCKER_TAG?}" - always-pull: true - command: ["bash", ".buildkite/utilities/upload_julia.sh"] - propagate-environment: true - volumes: - # Mount buildkite-agent as well - - "C:\\buildkite-agent\\bin:C:\\buildkite-agent\\bin" - environment: - # We have to list these here, because buildkite doesn't automatically - # include environment-hook-set variables in a way that the docker - # plugin finds. It's annoying, but at least we have a workaround. - - "JULIA_CPU_THREADS" - # Have to include this for `buildkite-agent` to work: - - "BUILDKITE_AGENT_ACCESS_TOKEN" - # Have to include these for codesigning and uploading - - "AWS_ACCESS_KEY_ID" - - "AWS_SECRET_ACCESS_KEY" - - "AZURE_TENANT_ID" - - "AZURE_CLIENT_ID" - - "AZURE_CLIENT_SECRET" - # These control where we upload - must be explicit values for no_gpl - # (pass-through would use the hook-set ephemeral bucket instead) - - "S3_BUCKET=julialang-nogpl" - - "S3_BUCKET_PREFIX=bin-nogpl" - timeout_in_minutes: ${TIMEOUT?} - agents: - queue: "build" - os: "windows" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - TRIPLET: "${TRIPLET?}" - S3_BUCKET: "julialang-nogpl" - S3_BUCKET_PREFIX: "bin-nogpl" diff --git a/port_oidc_1.13.sh b/port_oidc_1.13.sh new file mode 100644 index 000000000..2395bbf56 --- /dev/null +++ b/port_oidc_1.13.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Port the #544 OIDC/KMS trust architecture from `main` onto release-julia-1.13, +# reusing the shared julia-pr / julia-ci / julia-publish pipeline slugs. +# +# Principle: make the tree match `main` (the target architecture, already +# designed to serve `master` AND `release-*`) EXCEPT the genuinely +# release-specific build matrix (the *.arches files + launch_powerpc.jl), +# which are preserved from release-julia-1.13. +set -euo pipefail +cd "$(dirname "$0")" + +BASE=origin/release-julia-1.13 +MAIN=origin/main +BR=kf/port-oidc-release-1.13 + +git rev-parse --verify "$BR" >/dev/null 2>&1 && git branch -D "$BR" +git checkout -q -b "$BR" "$BASE" + +# 1. Overlay main's tree wholesale (every path that exists in main): shared +# files get main's version, main-only files are added. Paths that exist +# only on release are left untouched here (removed in step 3). +git checkout -q "$MAIN" -- . + +# 2. Restore the release-1.13 build matrix (rootfs v6.00/v7.10, mmtk on +# package_linux, Windows CFLAGS) + the powerpc launcher. +mapfile -t KEEP < <(git ls-tree -r --name-only "$BASE" -- \ + pipelines/main/platforms pipelines/scheduled/platforms \ + | grep -E '\.arches$') +KEEP+=(pipelines/main/platforms/launch_powerpc.jl) +git checkout -q "$BASE" -- "${KEEP[@]}" + +# 3. Retire the pre-#544 cryptic / signed-split / secrets infrastructure. +git rm -q -r --ignore-unmatch \ + cryptic_repo_keys secrets .buildkite/cryptic_repo_root \ + devdocs/sign.md \ + utilities/aws_config utilities/deploy_docs.sh utilities/sign_tarball.sh \ + pipelines/main/launch_signed_jobs.yml pipelines/main/launch_signed_jobs.yml.signature \ + pipelines/main/launch_unsigned_builders.yml pipelines/main/launch_unsigned_jobs.yml \ + pipelines/main/launch_upload_jobs.yml pipelines/main/launch_upload_jobs.yml.signature \ + pipelines/main/misc/upload_buildkite_results.yml \ + pipelines/main/platforms/upload_freebsd.yml pipelines/main/platforms/upload_linux.yml \ + pipelines/main/platforms/upload_macos.yml pipelines/main/platforms/upload_windows.yml \ + pipelines/scheduled/coverage/coverage.yml.signature \ + pipelines/scheduled/launch_signed_jobs.yml pipelines/scheduled/launch_signed_jobs.yml.signature \ + pipelines/scheduled/launch_unsigned_jobs.yml pipelines/scheduled/launch_upload_jobs.yml \ + pipelines/scheduled/launch_upload_jobs.yml.signature \ + pipelines/scheduled/platforms/upload_linux.no_gpl.yml \ + pipelines/scheduled/platforms/upload_macos.no_gpl.yml \ + pipelines/scheduled/platforms/upload_windows.no_gpl.yml \ + >/dev/null + +echo "Port applied on branch $BR (renderer trim happens separately)." diff --git a/secrets/.gitignore b/secrets/.gitignore deleted file mode 100644 index 2a84f4868..000000000 --- a/secrets/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Ignore everything -* - -# Don't ignore this `.gitignore` file -!.gitignore - -# Don't ignore encrypted files -!*.encrypted - -# Don't ignore public keys, that's fine to include -!*.pub diff --git a/secrets/macos_codesigning.keychain.encrypted b/secrets/macos_codesigning.keychain.encrypted deleted file mode 100644 index b277567688510d0a87dee84656d127d071925aad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35216 zcmV(rK<>X&VQh3|WM5wyBHx~pGPc+JVT?SGA;JhS^kZXM$5{WF^4dCxQRjY_n^#}+ zg9pd_OU}M%x(Jlm)YnYIKUSfk48&#z%VV~UjmueW@Oi`6CVWm^W%dQ|GaSr$wmGpp zDxql84dZG2^)E4~Wzj?AB|EBQ$WB}CSVDF`Z9O^$KpgDbe01Wvk#vsaC%F_JY<*O~ z&1Q<38VtBUTS9I_I|nn8dbKhOWv0^8n#Q``lNGhX4(PU+7hZ$6Kk#1H!(bPhgh`F5 zQ8+qX8xGmMoUH}e@As)F=%?x{l^)C=F02f5OcbzJT9d{ieZIvD1^9d;haZ--qfEE$R;7N6Yr27JQ(-yv!sit$Z2mS^o)s#*%fhNl#X=^5u(qs ztKHE5m>+`Gl$@(fohr1fDvNCQ(+AI^z ztYxN=p02dIGmcNqmB%JkKNi_GGh4ocJI$rgm!31VgT8T|NsI!ziX^@n$igns`pFFB z#fA%5+O~Xd~W}DB|ULWhY5Q2?stKxZkj-w0C znjYF5lHj$NBx^+?H~j7R0$Oy#6}d4Q)62>b02`%LU=`S2=-RwXP`8izT_2ok1OgCT?NJ`orHALad( z_w^KH@Ft>01y)?S;o2Ugs=Jj-h3@4w|MJ+wAOQBsPSCR4>mgZC9KM>im|8gl3~HL& z=B_%5CCO^x*+4P(hDsCLei^GqGlsLVW1j;-Cr{sdjHL1*slPZD@{aBlH%>N@qp^EQ zUhiJeZuCl8qG=w{Oq|Ws--|<-aK}pTbA|gmSGtLRRgu#PaYZqAEi_N?zCy4CRJ8$nH!!_0xC>7D{ByRx?6kIP= zh|BI~BlZ7C_4c`#1Fo`hNH7|D_RYwHeF3n1G)+O#(H}6**hk)%D%ja%?z`_w?^^(; z*B;6(bUZH_OBf4|tcx)uVHZzi2bSF*jzgQ!`wx3G_Lt zxU5l$zEAJW_d6e#$#X9BM!LV1sZqa%Zeba+l;Zv?8^~Jmqe{wJhUzPB4YL_cRzZVu ze|Oo^HZ%b}M-l6S6%dyYf>jqTkY zr#5xHUwdTgYtXmitNnADYoAcvi#sW1_+GHmlytRhvDE>_tndoG&>1`6{@9&1a1Q_} z6gtul=EP)Wv};Fhn03lIq~6-+;r8&V$<&YLJiPSHEbCzSQ^>n#&KTs5ctwSMx!Rycd=obBhvrD!W-o0m?#3b))0HswD)?`7)t7&R<~X&m1|o08!BJDEeH{$#lY1swm}SghL@=s?N*i zfOYwRD$_Of#u#9iP4LU6RmZEr5X=We^o$g~2(!77wo%rWh@&B>NRz9|YHR<_oR#0k zjaXgtO=E?Yywg8~*C5OuemhlRgm30E&sw9Z%a-xQRd0(!pD$?b+1`+8pVckyI3_Xm zp|pyRZ(__>T}BcpgqN9}ck=F$p9Ejum1l_GHC8#-y(b1h>8biYS}%QjpP{gz>4umD z1a#d$n4oiEObH86d4+&6RB{?j&(5WzxJua%Ho1Z)VdjR-5HWI;45bav?);fPSku}C z@$0!1TkDg2{3=E4jV#Lcj;so{KgP5o zA~uc7-PH$et!N<%ps8g__N!eF%qLi4jy5k8AKR*pY!t1uJ-z$@X?7l%@TZx;S=oeC6_ zHUOZ}A^>5VHgZ0&q|dqVrO~LXRJp4Ac&zm>0sDH@c=$cdG(_Ya7gpOsO)MNFt2C;E z&{@MYVih;R3&Jqk0}@8(kAMw03N0`D4vGP=LY-wQ^E&i`D6fC)S zw{1MMhqOID8pFzeIre$2rj_bCFe8ZKBb@>RQLHP7|G=Jg*{?V(4rttyA^jd%fT%mf z)gyi=*BY_5Q2dbcENa9O7l7hZ;`8X*1jZ2>?d${l3u_`ulma)Odrj%0H+3!AB3fM= zP%G49aNp|`4==KR#i zye2WlNMBx7X^Ald<#=mjhAd60{w2ho&_lc}t0FEMp>S=c#yc<1PkzN&%sHYJ@_S0* zYNQjWWpELPwIpZ;F&^iC043*V*>gKUVJc)(d5{3I0;1Ms&g3RQC5#ND^@R!&a-HMO z&h3=k-=baE)TUej1Q9x?8fw&WjrCL;Wf@hBzTS3a7));kHUr_-a01=2&E>ZLV;-Oz zrLA=Q_{B#5%2(tvh=$uIDUWYLpmC+n@mz~|SrAITf#^~V<>ntHc7^IrP^SIv1c0>* zf^6oetqAYA&CLP{fIwLTc5>$DuYCmitZa3KqDbUz*2$Tqc}3%=KD|>J+dez_Ji1;x zrkw6{>G$8NY;}+b&G8S~)z@fgQARrZhW}rLAEGR+3%8W3-ZtVnf2}9;deOE_X!N*J zuf8+N?1JNJ{<|QFi;FGQTZt1valRHGd0u2B*RA#NI3)P{+6e=0wHn=9;xf{UlrBOO zszP&;+}O^Ap@x}$>q#+}*MH26g#OekozP zbC1lF^XLp&KiH$N4BJ&3Zr%@!XNU1fR?=&U=Fo%1v$4LWQ{#&j6ag3_z6`N& z=T95NzV-nBqOINtiihd2*@hvm+$Na}Rrv7TYIe-|jDs<^rR&?@-H6Di`OePv)yg&7 zJJT;B%o)6J+e-?Bop!~^WURpSQvlfU)`wc+TTXGz@B5 zd>s+QWeXF`RQ}su{2VZN?s{*}HyKM;RaBp=zV*H=M};O}2sW*z4f2c%w(i}EktDQ6m3iZ3AhO8n0F& z!Uq6q8-vdO+<+hF1)p~Qq{A2nosM_|454)ghj!UGE4ACL9;ygR=yL^|wm z?LYsx(rAr6>9`@zB`SJOhMX&#utF{8#6E6MJb0u6Rrq_PSNo?@R8_=5YnDLHh%{pvdl< zsI$R=WM)VMLh3aVO1}&?grkN`xx2gp9^DVb4#&4}rf^>iCjR$F11`ri1@CJOwVaX? zp#yl!E!&e!9}=6=Hs*o()Qk-Om7vmTsnrW+*0Td1$FDJ83>)ydYt1w}O@xc4$u4iw zi$DSy*l|VvZFQUO9(%scyFI50{HNsoQGt`OsFq0DYBN;4Cv-QFN0>=|oc!NB zew=&7_%Y)MFBX+X_MwD(GPRIZ6lmsN7h#od_=1<;*z=Tw#MZ+a3U(Ie4|vk2SMNC(rtjhhkvvBp;ymO1QiepYtu&|O4AD_798Z)mLf7}dk0y3Sfu zPxJtsTo*MJe3qm6)qIa6l#u7q8ZrA?jgEbBA<+TK!_rF8rh7}v^6tpL(GhNy8$z70 zu}8k;+JtXp<4YsoHK(X7g)k&%3Pw;hMvIx$_v!I!?lj{v@UR-G045>t!+6s1N&J4$Cqmyg#;a6N5$TCN1vVv7-bac?=oSUpAyV>(6YJ{SYVZazuZLBbW9 z$~?9e6bm~CagT^gf@HYlt{n$4cJzW(i+B-1cfR%p6jE52RUO;Y+$9~si_RD&r*kHy z#%#n(Rh71Hp?o-+%|XPT8&t!~eTR8#ezA763}_hh&2+-`EO!JGFht#ATXh8Ls-$&o z7o76C*)0vFoy}$h+rdpOg|6Hw3!5tSu{ATw9ITBeiv;xCC=U&>3xYZ9U|~EcQ>?B-~gN+a#r_d12U}w zGw$-EGVV^FuG`N%fEF&Af~q8kE4laP5vX8o<6$y3dh`SA(cynRgG$;EF`=O5sCc~R z!fJsJv9d@%Zt{gdM{El0f`#lIR*tS59^s_OOnMBFEzt0_Ek9;AM=oCf%h&$m;uG%o zBw}zl;dboB5Xe?orh4oz82B;O;$=7Q3jxE^v5m*ls8KRzVqc6|0(y-CF6j{A>W_hN z%;)wY!z=Th@BK0V1WE!FvU5GMCyMMBB2us2Vkes!?YvccosMeB&?W1d7acyo4i^`g zlCZruwV4mJS)X_+oZK00SQL6M@rh+M9MdUo7UlWJPO`N>rnt&}-fQE~5H8IP2LapK zdVSsym+9U)yxV}xyxrrxM?~@9bfKv~+EyYsw5*;yvwkI>;?FQ%MbG#%Tv3YS*Z1+n zkpiQltD%bFe^OMYKPzNU6I(5}uR_P6JT#GD3ga89ttn;bI3UE|siD9nZp*p|2Yo?r zi`X^~kz}6j57%0DkC8uk&Gj2WP5lu$WIT7= ze}k=UIzNFUL_u6DPjlB^zG-YdS=qAq>BT~6(d$7098N`EO=7VBNq9jX!LEhWj=<8V zzgar-8psyYcxeDN_n8l(f7S700S6i=@g8Z5R7ofxJ6byH;bCl+>=mp%zn_R~+1}vj zB8{K*E8_Rs`%sGqsR<2mHM1V=`Q9B-h$2s}r!LU6P*YHe)f9;hx{mRIx{z*9M=dz+5Lsp(c4ROAy*!WWNz^HafGLyV<9O2xX8cm3 zzV;zv@IfcC#r0tNnnu?h6m`$i`OEp9x`l&t2Ia*ccNMVL;qO@YZRde8@flN`fL0HSESW%H zA$ctqq&*u-eP;c*mgF^8t+~e|oKl5{(KgTqN)4Kc-DLWy;E&{ zlpB(qRQl3_qgFu^FESq-OT}X(P*r2*a(zliD|AHPhbZUF9cL=Im7BN?&2!!+{=-BH z0xheup|k|D!Rz4UX0npMH9*pcps|9rr>1iex)~J*gJ4q zM^rMOozU4dspQfx^i@!t4EMnI1ys~hNh#1Y@|VDnS>#dkki(d4?zfzC&IMuP(<+VMd4#k4!jpB^eOR4iTN>{$jT6Lz~q0dZ{ zImVFt*BgmIc^$J0ZpVnj+Y)x_Aw|gktj^EG{@}Gw+D@$Gh6QU`>`e)Nz047F_74`E z)CvMY-kot!Uug>zXl+&Eu_P1-mW%sO95X4$UG%W&J4WW-o<}0re*8J$g9gCLTGl?6 z&?_}eyLE3orK0svIyv`sxyf{vlHZgON;O*?((&-(iT57UM-L$yunDBwH+a7y*v4v3 zfFPUgh_j1U*(_TmTuJ?0U4KX!tH~S;i)T&$2F|CRtbs>x*`xAE{5e-cRkLGzOaWi)~gb89DNG(}|F1jK;>h-bL+Yn@yjJ5>| zz>vcdACm6dfa6s0KdIerX!VJE2|_A_pv-K1jy?$J%LP$ai-(J>J2-qUPVMh3{6O87 z9N7`hXbx@3tZmn&mQEw7 z4h-;N+O@eSF{UO%1?W0ETL}gL5rr`L6!%bahs(J_WD}@w{xonpzE8Ji7=A0>ohsN! z7(}z6yQy(=WUAou?vR&xsXG0BodHqb&M8fZ_Zq1ny-b@LKDpeZaKJk9S#%eQWQz(v z9D3LKa%}z>anzIuR@@(>5OCQL6tQwDioY21XDvdfX{4t-G!If&nU-;v)Hm2QiYWrP zlzYYUE_rQIKFYwD;6bG=4_C(yze&)Cy_cQnGApqRU{5p8MEBBD-eiug#FRXS>6?;l zY-*Js2r?+OaCF=yogBx;0{BAhtEzDNqK2loC$)7zxQPb8x3tu}ehQ)2fHkCtRs1a; zMaP4}5EQG7{&rrcp2R^{-ELVqx~jQb9#3egN=ZkosN$c!LlsG5wmYX?YNCuRGlmE6 z@FJ;(ck@jH-CykMJH$23XIv9J#w4r8KO=^P>j$Rbq^x_KwJFbb(;i=n4?Xtjz4V1r zxH2lbaT6%YW45uPNGwYllMHy>3QN(_meh{}fhlf8E|QZJN-3{t%z~@s_2u@d^H+g) zl-U+`f&ELAd$!b-wl-hky3L#B#SFFz#k2)xkAkHD2$&ETMQiJ^YIBMZDg0K{i zQnlsEp2nRTLX@(d5_8Hl*`x9n=!)wl*qUeeo7O&O+FjK!`;vLNA!vEj>IYI@N6pu! zKlflOP4>hJ9d{jn8i7h%Fgq~B!b2m)$bKr>KBQ4?{YZPd%LGbIkcB!k7LU^~7&osW zf;tpYWynxQe)Jj>U}<@V1Vr`LTBvJViI${yAC2_kchfuJ^?>QorPCsqF$5vPz;%d| z#}T>eOtA8 z8G3NROks>gf6TcoX5aSL_46p$t9W`rp-oqJXS&Z6%hOVkm1D=`u9Hzuw31{QBgQ3k znv>Fk#;4?>=pIq$UX+%O!GL#FR#1qp$Qsly7RQ4?HcTu`?Nw?Gz_rn1d7LL%&gUfk zEXHMfDxjOho6~Ch2@b&BRI%g*CQF>{8JNUg74!aP+`9zrpZ)Ex<%XoiL%tL$fjGuN z#o;qgQ3$0_`&vvL`gYj!E_G~FvB;u>?gmYoBB6SApEDf2iJXaLF*$>SK(09Z`6?l; zJ18qmYmA^sb1IyQ2JQ}ogrI`2jkitOnuhJGK>yCS`&QhOMm=CrE#TLxivTyro;5ib zpzgA6kM(>SL6w52L7fLz%5SF-1C|@H#$)@HpaR^HY}8#i5N8?P%#v+!>}t4=weVP5 zQ~qT1wiZ&V^|NC=jg`1??16HQs`Yj9ps1B@t5jq|jF{sX?5Tl`4D26IQupP2aY4gN*=;=Huqa>iOPBi-PMukl?-YwGYEC0nL+<7I-fKj}-U zU<*Wrsa-R$VC!8^MQGqP2rSzJ#T#9W;CDNCNUOHPYuhpfiqVA!tmj0o25wDJ5V3M8 zvkXR+d}h-Q1uGuWE}FhneCJx(g^=K{LPM(D)>l3To-rTb_24b`uA5bymb~e1X*ymY z9-nSGolE{Bd5u4;AkAXn>ETpv@P9UhG=m_b2ZV)T5e;;dplO8Da+F%>W61(?WLM14 zjQE$ryz9^PKQp2He|&fja51>7uQ)4zju8RNLH+r(b}E};`gV*H;N}MENjJkkP{ed| zVUh>%Z`Z7HV9F}(1BTChVu~R)&t0SG*4L}CIyTPR0xrQ-TDXA$WYZnXn$HpgII8&( z!vDchwWp1R+5&iFOj4cP9mu@Z6zx;H+SPJAnmS)?v$@$-kJ=eE_jn`}amQY_`5bV0 zw3&S{g4j7-THEmRS~W1xp3U-T$x~~6zyQS*&D|<3zAVZxI};#(g>XzqXHlXQz~f|AycNCbQhGjnd&3CY(er~nNXo(;Ux4@On794l?QaI5(x>-8F0qu>2Y+; zn4PAK>cxGay%?_d-|1Kn9B44y9w?r}p^|3`NlRf5pDt=p9bjsfTEoIP3O`49XgSQf zg)riTw@<@&JB|EOLbb)=T)bP?wzUpy9V%nM1;Hg+&1bX=tJm+=D|K#kdJN|iQ*`;c z4>|%FU`kCVQ&9}2*^0m>R3EnMsEJoRT@6waSyI2$1tEFY@v5M6g>hUsEgh8DF+fBQ z(*3o+FQ_NP=7iV0m^6|z_z&b>o{c&GK_4H0q|%3=5Y*9^27(3N<`(_$J`THnI28?x z2ohBR&$I9r8*0dH-IPy;Qx2(3-N>UflKwkkhszPSF^~ymU1vth4StF;9qp+!C*%U9 zf}bVelXh8AbyZLzBKGA#g#(F*x>|_Z!WsBaD}DrS8_J!T-UXi63HGYgG=dK9BWO%_ zGG<_@hU?+(qE?OHO)SBR!zQaGkYm#1;sro2hBT zt{W{ElwD8g;)Re{qK>6UKmX6)Utct>v+kRw3s*bku{HZ`qY4Exw(e1dPHZvWa1~9e zO8k20-3d^)1>WaUk3)*yjTJS378U}kf_H8rl_Ac%zm#CMq@>|i3Q1|CG|!!%nR85( zD?YfUFAqD=NV1No3SFIj^F&e=X@zPR=Wc06Lv4!2NAHc9jor(nlCI8Q^Ec1YDWBOA zc9f2Z2iH&%kLKg8;;zw7vdy;I^k&2;2~P2$=G-Nr^CN7Mge&PjTMfWX_0}pU>&C+= zyH(oM$dGbrNYoXPDZk9fX4VxCze@wanI1H}rM6oXlRI);SJo&4YMZ1cuu_5(-8e3W zmSM&1RB=Sgd7gT6oh%mba$l8IFEv1x>k|@l0(qUHsgq20>E!(*Q}ihdGc+=;L@L5b zBy{wcuut@=aoaM477b?TN@Cj)#srv!PA#2x!G!(#h6vctqHyP}?a zu6Cz;M%9nSTtQzwqobZ)7L$R(<<{qeDu9QB)V98!Yz33Y=983af;Vw8lvlRITdSup z2y*6PP5^AZ$F!O-7^8E^Hw$B@GYTxpd+XNU4u11R;8O~463*70~ zZW2W<+*o+T0=*Qua-(Q|-4=Sd^#Hk1Gc=ij)#Fmr z5hW+sG6%z%wDGDV%gHyTeTJ&H<(pqG`~-us*hh?wDY<(*R86o&X{-u)Asg0=Fd^B; zhS7`?EkAzn1W~yBo?|LgDJp{sYR>~!sqv-^g$_2_E+vj=BHU+bwDl*}prwQxan0y~ zq>^Of(H`CKoEZ!nGx-7)+T0_{oQDN#RGq2OI2znfF~+VhjR4)kpIeyg+p?YpuUGs; z3k!7X={Ap)?M?j;d_&nS+trYDL%b_OGC`c$xCy#IAnyDXD_aPAP*Q`O{**%IOojU} zn!a$V`j!!msrgpmAuV|}re#Ax<23-j4Z_#sk z^8*Z#Oq^kB@MC}~uVG|=k8#8;E z>A;FnHwaraz-s{SY{7m7Z4P?=93vm5P3x3M@v%TuPp)b68d+@w@lERxK8*YcZe$me zOG8wqJxKK^)dmGH*Z^n~Kl&RLEfI~iX!f3;_Wf`@1p-m0e(I>occQo_@X`sSX}GW_ z)yC!=*8$W{El?y0=UwmzFyTWbT2XNG4@3J)FnLThIAP4pxXHS|_@6)m`Xck&z8r>|r15H0r9`guelk|41z6xF;n?swHAtZDkT zLR745V5qLvE$(`{#O)gcPCE=(oh1QkxnIT;Dzc~WcaWDk62?R*nr0Xeie20m)_HK7 zbSnRsS%($Tb=(ZAHsY&v8`U(}hoImZBWEMcakoCl#lK^pUgD-JcI&quJ5|x_0(s?S z!rTQg{%j_LPQ*{qaD=zMC6Xq`LWUc4UEV-h3ROF8jJ)F$eic|XLX=!P88mhIB1ASW zQ?D!_w(K9u*?U8U|E}Vdt5=7qfy4#txQe5BjhqrC@fzMV-J3PF%<(YUA^pq>xG! z0>DTRwdS*jXxB95XT_2h*KmrK#JRcy1JOg4imG^#Lz~$1_jplff}Lm>rt&4nwWmVz zeI*uNT~U@Zf8m42%s^V9Yq4P4Pm0~_M!?~Vgz9s9Q$~%J4-j}bX<%5G;Q|YkW3qcs z`&zCQPz0>X-#T$&K$U%nHe&*f+j<7m%%4Bw%k_|70M#8NDMk$U;g$w&_vHJNqAs+|0FDJc)jM4V7 zWulwuZsll)TE@Ed@Xz(vZDb3H^`~0*vZ%Wrb7>=;lpZi|(ID;D%~=hoP#3TZwSj&n zc6f2ALJ&co7@cFw2V0b6;z5h9Jc}KtU^ibrJM{taNc`JZxaY{On^?QDy6~0zV#+FN z59*IiY(L+8I3?=SGywH9ofN#ukdC3KLX1{rKAZ5pD>9{pC8@ zh~&-g<>-QqYs;%X};P-CRBhsfe){{3_8BdLX zHVkN#9L0eHtn*r#h?emyx><9`KG+uf+{yKRXmjg9e?-a^dzCLb%r68FRGwDH!C>ks zqJyUr$(x-Au;)56wh+l8`w8TOL_M&tlp@di7wy+zQHvL%{1`+pvtq^O%IX}k|xnFm}@NTAz-V~$T5{{a)8J)pIlVq9*hq{zM$F6!)E z_=E9i%lC2;EWUo@S)cq~^ z-08vv(kFieWjLvp?rZ^MRl}@TdiC~;lmf{w?~*MdX>IfLjW(Uph}k@9_@U^kt`VfF z&IZmdbBNg=Osegg^N=>=J*B}-YS?3LNbW#!Cp4LT;WkwLp7WO)qkx+n1;tw_?8S5I zKIeU3B6m-XQWYqtnuMLp(5>y~!r2Q{kfAmHs&7hXg<2868~eSdez!-|^vLWP z&T(Upv7dPqf*FC%?eOm-lEXKZh)rnW0Pw&ROdY8rk5}?=ej+QnbTyS*Cw3*d5Qf`~ zlBXy#aRMV@F)r<>L@2Rkc0Gxk*Zph`XXIJKl(RsG7YN9MZp;nyC1l70=~|7(afN-- z4#YU*MEeaDITWXK;)e-hPAsaTl&u!j84ml4gpjlT`eCZc$h+vLG+L}Hbkzb`Sm^jdKav-65$zF##1#|I zll`-*G2ic5R1Wr_d_+K{t-G(835T&sdVqU>+WKxc^KXZU>`mj5_Woe{P(>>Pe5IDH zt}M1W?emcNXni5NNEH{YAa38g&px##%Egl4J5>}30-r(1Bw`dVlPr;umNX29e%JQDjISO6a8+rS7h=UaeuYeV zFQmos5AhBlyM0m7#j#MPAf7( z&LCtWA?H@E$Z=yBH|9D%qwKQ@t8=V&ERdaVTTl1PEr|W3Dmxi5@*ybW4$u67mdk#I z!18O+X%sYTRz9mv%8KS?+>rW)z`6=Kfg`ZH7nds2)M_|{|8xzyMtqEE)$r%UuCuJA zqG2MLq5QOyeEsRFKV-oWlSyt*XMyu%jath@o-VWo-w;+JLB0lqpwtXVO{_{VYslii z?U>f_X;4O2>PH7Fbj6CaukLgm8os}KVv*6H?+zkQ{^w|fy8qxZY6)JdY`Txj8Y;34 zyPhl5Ii^im@UiCmObtgVY&qZ3$Am9R%5W%2s$;5l5MXU#rah;`Dc%_UO1+*N9o8QM z+e+wd?T>TWS$L_`lxyDG&+1;D+#2+}{YtuJGEH~u`1ql=$eyplv=L3leoa$X_!{cB z`k~u89WBw^Ss(vIrZc6$)Sd4s;Kn1t=y6nzuJVNIR6DnJp@faQB7?r;jc`k#kKwyQ zx@PmK^xX@XTisImwfAW!06XL)MBaj{S){yZVPNP7iBEy-&0rQXE=QNdw&}N@7^13% zNW%Flq$c#(IR9xJlg#-m2dx5`n>&jOFckCp6K0Nh+yO-7X!S0V4LI6vJKlgwCsV&6;^78q zPO-MxuC4~YRzc^JlD`oRKEH!~XjNtg@`7chfukZNN_895FzK_bo-Id!wQIp}KV?_e zy|<5;4BA~bM!~|~(+?xu-3NXe7gpk3tzXh&elUb-eo=y^Fn_@SAs9eCvTJplFp}`# zh~2Q)AMUj}>Lq9ehx(5jSEn|31L$z@s+A;B&EIcA#LOpC6s2WX0T{Oo2V{z?KP(Q? z{}m2+hvTb4Of^%_$Rcb2)Vc23X@K26=H1m(xJ@OJX2YRP9p75RUnIX1s>M4JBC!3? z#d;M_FRcYsBA`UNq~RwbXC0y9N^r^1>-JMZg3%V##=om|q%~1fZMe5wCs;B_cqL+M|E>GX6!My^xd!Rytj$O5LQX*wtHl*FQItk){k>npAw#*X zZWUqidTC7QqO!w`*>3o|#NeRia=(t)C(#WU55sCdm0-MYJ-&j$cov4|F7bp3vEkh; zCH6!7c!s_-)3R(}c!v&MH_BiTBv1^N!pxKS1~hQ5s79=FH+TvxG^RJLVVFfKRX2&a z9{RmWF203u@NU@29?JOmbId#DsU7&~fV^%aD?$I_i-ET^hM-s<)AH)wOmw^DcNvc! zD8LnAd+*>{Ge>j_0{%XQDJot-ZF!uIw)xS=K?x?6@&Z~hiuIiFTAI_ zLL%*xL6~Hp62)^t+rO)6>uVGs!fxTkdU^ITOvJju?PJ!I;@BZXQ+`cX3s*~-O z6kUi0l&t|6xy|$%ygWOkon)f_BSB`ph4JAhO139=RRMoQiG5|JO|XFN1qfJm#7fg|l9WMCcwu9|^& zn~SEof(aw`ECIH{#s0!wMSk{Ow#$C=Jv_ZA`!IxuGY+3avi8i7jVeGmw7LLjzAMORl)R$r)p_!>qvNQ*HC? z@0>RUS}`IiWNLI1uKq_fEGi*_JO@#a+7D^yi+nCCNCGcGRhTDeOY0r(1j_8f84v+| z(d|aF>FXDMXZj#|sX9tgMUDoKcWqK5*f@84xdFRBn;j7#L^1vI?+V%KOF<%uMGjShFYhD69(?sWx z^!Jt?dwTTb(RtG#ba7#zAJ{S9>HrZIMQ`NJDCxRb5`Ai-zZh`k-ImXsP6@3vN-Hu2 zH0a~doJnxuZ%fAenknS?Bx5`s&FJ+MnAX8gASrS`k#eF@1UV5ESKnKvqPx55uQ%FT zdsE9|)*9u*URhxcTo_6N8+CWJgRSRMPJ{J+re?LTD;$vY7hVSq{_OGeC^p7FgPZf^l1PJ*9WS zs{aFD7#bZF&op>3uRg(yt)(Dn@_Go0pKl|uL(_OYxo{tyEws|ueq;LQd*B)!EQMlnhDoI@Hxl*rB_8+3i&Qyor-!>#CCa| zMap9N6O$>qq<{;A83hK_fQxy{#0iVri_hF4vbb2i$!iQgFvGGL-Sbc}bH0Qi=?`E} zXXWl|nGAxAQAHZWLRQSPO_s#Xa1`>NI$-HDzzH;K$FDRmiE4swhs|X<_bDMq`7{9s zAV$FbPMa+rd}c*EA6W@O!NCJ@9E9^=KXR06|7RIK!9lrh?Km^Mw>vH(uS2|g`|)_Y z@eRx5Dyk1R=RusiV=B3jiJ>e?#kcU(k#;o|RuKN@$hUl$!I~p8ai!8uZpyaCAq`|T zV$lZ>{LXNf(uGUroFEyh0FY8vvu)r2!R?1YW1 zYil|PRH3Jkh4s6&3_?Av5~|k|@|n7VdTfq+ffHKNa~hU7{`iKoz(C^rv==@2RB}no zNf_NmHsxQ?- zZ3k4Y64rx_x66Z`?UlnfO2PuWDNyC1T8Lx(|3{Snziy7!2RyKF0it8GpWA{XcnWDP zF@E(c$V?r6`09BH_Y^z`O-uTlzaEXc7=pDdDdMU%L!T)nJ+B3&-0`m#KM~jaNruvJ zQ`Bh_izTr0l(CmQyqzV=CDCA(qk=REC)tMVSd+LMkP?ASUshSaGw-B8KM>s1eX=XI zfW*h|`nRO`^_7HQ?0DE?{(z5n*+yQ|FUd^qyckYLpC?4ZurmZE0uqd!|D5Dj@uZHy zFJN{IZD_Mm9?wvdj^WiQ@ti!xkn%A*BtP+ky<*?j)xqj+6x}K4XgqKfU9l=;ic#1! zf?FB9iL|HmP15N=PCWsL)cMI_Rt0=A348JWgNjd)|1QA+R`65yAS#~;)B=LPEp8Um zWoE%K>}F=QT;%cGT%%aNcT(KxI6-Msuxx~uieSmo8+(wXa+{EU!cC?Wz0yTkP)KkbZb2{ zkSV|2eW4Ni(=Ll&9MwJ1Gtb^{J;I?16+=noj}R+CJyJS**%5m6*#Ju4_pS-2l17)? zr^m66$e#2zq}w-}V&wX=ImoF^nyx%WA^6mWNMZ<1hWK z#Ua%2{u+{=SkJGG^+ZC|+Uv-aCFSfT`ev<9Swr~49<3Lv<_06ul%yb6t)f33F{37K zm>jLKX! z&`#XgQnL2?Whf?_P>e$P)8Xhd(S37cta;ekZ^N;>(A3iTg25_D);+0^-=v?Z;~VEY zzIH1p=ub%7TOCW$wa>S60b$Y})|`A>V_vE48p3hq(~?hbD*+o4I`TEqlfaueG(Tx< z3Rh|Qw`-URX1eyK)K5)BH|s2CCh|}zI7r__T&SXPWsPn@%8Z*xwhx`_HVsnrWPGrm zxiL)B!l;bqJ;xxZ!3^+RkPQeM?0dcw4=IvYA8A&IdaYMi!EO$CM~T)wk=g%2wktv>*QKaq3Q@HrK~vV=GZ?$D-L|3+*CU z@MZ1?F+uqp+92jWrOYcQ z4hZKpK))DJ^C+*1kK8>a`H|A1QO`Q+n?B}5wjxkge(Ct9JX?%J5k=&Ft>|g@P~Ltt z>M|704}Z{q;oUwmaNsedC`#@Sve>P|>Xg}OcqC;1&iXTnO}viWJ7QZ?xyPbZUdGNg z^G9F`f-ynIh2+DNwW-OgfJYq4j${sD7o zxMh%TOqr?Gy$s5qw1v4me8pe8e+WdI6hDt#`9IzLrfH4}4mPFtMI79tW28E8D?~oI z+xS1s%_fNxamn{V_t5uFiE_ZUhj3CC>wt1pc%m(RE{ifuazgGtIOS!j^mmFDfSsZ4 zdqQhL1;x7}(q|sJJ}7rn)&pg13`6iT5=rd)YUxJ% zLtMJQ6~-$pU)%r6p(xRW7xZa6$~h_ZlH5HKZp&8xJ3iKTDtXUmAe_{HCk-LsfRZBW z7^iB{#fo9HG7_1p_D$2fhn`qq3TUw(pShS;kqflA`()7u6o6TCV`ZPiOT_j$>ur%M^PJL zQXidNS*Dj9nY!xx*1=L3gBO{$ZomfHev@AnxGQR zle9sYCH3ewQ%@mQf76g?RRXpTUn?~Ov5#Kf04YG$zq^yjS#5*4?LI5P(VtH?nl!EN z&7-#fxnyV`o0lCWGHd5wy>nC4VG1Y+pCyaFr#`61SK26Zpxs%(nxP)WoOpvRYF4y+ zH58Yk?$hDp=t^HO;jPAJQKo+QII$B}y)Zz}9!oTB&tn2X9uzkA<1wn6{iPogvk3q^ zp@b7$b%Ns}$p!`c`C zT)+B@Fw58Y$2+&B5I&`RNG{LdxkUT=<>rzxip_(%WRPM0x7dd5NXy-Q%oE;ilAb^2*B~ z14FINeVK%p=lEsTlAZs6Y1{^*e3<&_B+}NE=b2JNiGlN6+SPVNJ^f|tRp^{MYBrIq z7r-o>Ulr7(PNSbjECGDM3*JfwME0377^*avDYUsRp9vYA zgozBMadZ2LEwumf-)})w{T26^acmy(cg9RF@$~|2Gko2J9K?@z!|pjUCNLU!KT|>s z!=~ks%#()ltChL|` z4YodU+0V-70cdMK7iaFgJDFL^ux3{s)&03zGU68AF`Wm58ksPPV}q6m3-%eXfFaF1 zd*>b)e1&94eCslyNY>aw^s@ySbOg7=>7NLv7s*bLfPx}7Uvj0nm;^Tuh(zmrwiGZX zwinKzvbX8)8*DYC|MfB5eW)1&|9$e(pFr7ZGtt7-)tWg=Q8duF(FlCi!xO_#`Z}#km2`(QAYP)1GFq&eHHh@_@vwnp2HeTbM0wZ6#+Q6hbooMLq~|es!Mas8t$)# zEi5Cyu0Hx~V6l)mT&Z7&5A5BLk2i?AE&1n|&t7j7r-WXAH)D0Rx>WvF4L zFr8u{?8ibDzq;5B^w`W#XND^)pLUGltvQ0}dT#v8Dt)$H%DtBVtGLIni*nr$0`@&D zJW~+{!%s+(%C|}qyxW$t-|I?dF20MIa`=9LwZ|tL1Vr+=mbI8pNbk(XzAboLq)x6m zMT7Uhyd+aI5@+owW2yA~b9I808}fR}Zg;ynf5n0a33w~(O8ZD8Em=}V-lVgY^TX_t z3!a$c&XECrFBk1Wm%8Ls^lw}jjhPD9m(=45M-l)unG@SrcCcJ!YeT-A(LSup{R(3y zh$8YuJ zk874BmclaHe1m?wq&h6)iCrk7i_=1RlddLdR|WKK=mtr*O5TCtLIFX2Nsx4l#6F1= zuX&N$lYUED(7i9x*2l58lN4!q|N10+ zq0sTxDo!+Co{r4obO{IMxAJ$~K~4B9`zzw=FerThileCO%IZ`2pwttafxbpuJVC6|E z4Wx74%G?RxkU36_3CcuT36*E2&|@j?hCWe49;6*``mAfZ!UCZeZ`?NNJD#I7=lXY; z{S$79q6HE(KF&jBwczXEyzVndtD=@G!aX^9Sv+Ci?-dVP*Blup+f_u=* z1U{}c^^`F&UUtCxxKV1ms&iX^JWO^mjulcm=t*8#`B$ad27Uo1OsC+o3@7e)iaa%) zKqPMH?kjdE2G2Qu@zS>G$qikepI81LaVCTq5Ifj;n@*u$vR45YIp@S`4dW5_)OLi6<|BuR4#3RUqW)IJc_967@sR01kl@fN7~o8x6~H zU3o4`X4V?%rHivUd0T$_T{Cd*@@(65%~G&B4Ub2kZW#3)VP_Q&=uV7Z?mZo!L0-B) zmjfC|_I4qolV2K&GD2xx$+wog& z#0IXxTlh`yB-Yq`#ptD^7+jWsWF}w^TMOkrq;(sn!*K}jNKtt-h>xWRIC^7WraBrg z#bx8Z#2yS3<1zx1Ef9FtGx5fq7-GlOXI-2JjC#^Nf8sG1o2dcb)OoD7=#=G#Fia%D zj$i=LRtVxm9sqik#yi9PeOa6`6jJ4(23~-d+hM{+utG z>mCPr@p62DDvTdMU$`y*qeWU?kOihkADrnB^u;+ogIAGMmPStbpzV+v$T^D;c2&oQ1N0&u+Z~3bw;HeeTkZwiN>VsG z^*)j5TTYpMS@9=Wi<+%o9}I5W@$cts1g%)i#k zv!uGf5)KqG>lwybix-7+l*mQeo9^rBhO)d^+0l2}pZWQob{Xd%)4hzY-X82LtV*zN zYv%^PCUSKgt-yR4I364>A{|$iHlzZ)tp06MhP+fkgNzCDgLN37pVIBIBu6#7+mbZX z43H&8MhYELbnTjb-8iJH?(5HTg7oj6Qc&`e*MD>yt`2lpMud#9^1y`PpTOwqt@0$4 z2b*~rI$Y6$wA3sK1WC8UmWko0NmI`9P6>KM;7{(ZJ@|}Nyi;ctxPxqs+9dcef)#Sk z3kK1ZT-AKGkbx&tm12e$iF;e13}@m36}ym)&G&j$8FePx}9l%|?Dabr`r#fen) zbFh00SE<`KjWp#5GVSbLO69DgI}cwWTw!$7fCq zTOP4%{>lM7Y1grIW_pj-_=zMoMb;&EP`jD zI6IJta?*98y~VTBQfHV?1Kq@{USH-9c9I6^dOR7Ar{z=iNh=@2mx0tBr3^yesVjK( zum}_4ntrRrXLL0{R*_n_ArTPO^l6;l&jV`A=8LgNvpWR#TDb?9mo~_;S^G&Quc zT{xoGBg0N_t%`6Yyh!wx&7OC!D~>fmJksi>Q=+_1`Nw>w*p`1`_(G}Vhs(FFW9w-q zdZ+1DH$?UeXQ-dI(bKmoK2p#yXB&W|D<>HK5$ViC zS}KL-(LlEn1-hou0v~cstuOD{bR!`n%_HJYRLY{A+{u@vfWnD`OyA*DFzjtf57vH^ zNx7d*xp=1ykFQe4yn9X$x_7zP8*nQL1UoT;^)TT+-g2N>CDJkr+V(9INwL?OrtVFW zEAW5=Au=*XV^YtoV^P>K(AuTUsa94&zME@T*vO#}U+oziob9H5&jcHPf)R6@lUtG) zW$ssvH@Tiq%st@Xhznr8C7`7p)xZ*sh}-=;uc*uMj3Up2`q4i)elC!7i2wii6TTT@ zl!hhwT+wj6@`|6b8H4w#qR+2K;fh+Y)jVkfi51#GE2_XId>L*YTnsOn4xwS9Lem&-TZV?ntFUGutc#oF9PPW zhiULJA71)}SI&(X3PGR<^CZ|DPexX!VJ*(jpH{uib$tXy!GjFj2|V~~5w3G^Gxz`h zE>hqD{-IoT=qeulo$5hLz>(LTeUG|3hzdt+4gXFit>)~^T{!I`HHr|3$;-Yki zhNiRkB`1oeN2$}31{u%>O4Rt!VUUgfMF9{onoBv@R=OJOD|d%KALP^CvLa?six`Go zf{Y4K=lf-f`1`m{$b=sH26j%5E4s&Iuon+G=~`;2;FRQnwsV7601X{w?;}!394Y;S zROo>eW?>V{<4adc^^C*J4vh-<}UtiHd7oi zK(OhAZTrU@>afn^y_$(ZcXOuTY-0Q<2hTBVF!D7cZb4kIo>Se>t3+>} zoXnOKH37b@8~hhV0^^5xoIj7Z{V9@%!bEVyS<`DJLF?-u^8uiE^7N4Ru4a4)hZG&K z)QGIH%OuyB5R&BSF?b)Bs)1dp;$QN9f(UTRWefthDUgxxLq=-WLu_5Q^Wa;ug~oFe z76J?n=o%^eA#MrCCD(2G`UGV1deT(Yq0L;+wflV7;p*ylsjkFIDF9O|gGqijHS>hd zbt{|Gmmo(&-n;W}$o`dtR6n;3>0yx~N%Z7G$T7sN3`$~a2;pSEVU0&?EGNHeX4vE8 z7AePq-COVqvxpb=Az#7{od7!0j&ziW`NBojTFTa^y^5}Zt1F#)QNDf8yYvk04lT+< zG?<9(X!ITM&zf3)u?Xbiu#5P+{T}7qU$yWpyuUD`=o?=#ILuDRJ%<(4o9%hl85RSP zdPC@Vl52w~0$r~x8N2uyQr+d&Va<~4(pO2|3zyhU_RXxX4M?Z{EMY>f;-eKjorgS^ zIk}V&nA*=fnEx8K>Ikhu16vo{$B~ZGK@uu2*d{rG(ElmIZ>@?uoK0Dz=MIo|A2u&q zHoALA`7b_{A`ql|dce7I9CZZa7#*;*!TN3Jz!@ zPnA(0Mj0FQcFibWzsBO_C^IT>ANii%ul>@}g#s90^-vOeP}pz3CI16h2F~YRP!oX+ z_Ck>}q-{kvzcadzJ0AepPWI!{74}u0-J2oINfVnaed{%ex^N!>agp5q zodl3@hZxX@9(jVe>7E7qg07w->p5IxoKFXH+q-2V8zSB##jUwpi$tYB22!psYDL3A zO^h^KAtJrKy&eGJkT#pl1v5@NZ8H(@MG(nZfuFup^W~H^jtiwixkGp_S6%{kcv9FO zm>T+AiC`BtD@Ni9Yxu34{*f$yf4-rx5mUv311|cDevxpspH8$yWGCm?Grg|eDEVnn zf(f{aU=UFYNFg;Awa77>$e*+trGZu)(l4?Z?4+!)87hLXz$9+}LwdQ+4K8Z}3~U%?6@DmEzAFcpQI~D5yVr z5C15QjL8#nI@zz9ZYzinyD6uPPDSR|Q80>Jznn7gyZ=U7syx!Dom6Cj3DKyebx%rb zj0yUl-zszcgaLxUTFSj6lCyf>!?i=pf#QY$@ojGQ-U38!{ zsX26ImLosjo@WW-I>QR&g_y7UL&z`W?49-HDbw^#o2Y8@wdq36Zzp-56D zmzjpqFlgU@MrCR9SsEcR4qG!jKPs6KCTb*9rqfDRR+A)$1%8*yZuPMGi%JeLoBsy;p>?wh!^U2k4Q&C4a^ z#?=~KNk^W5b;+IC^pIs}N9wo;>mY_i-^!);IEN2gVTUYw5(TRE5-#R)yklx9B?Qu% zw!jdpu;xvCN-!{_2KSJsgKM8uWQwm8QO_03lO7QZ!nZRG3{I{NDy%CL{h|ZW*iO-b zJ7DvL|BsMgNn+&TsQzm3U>97aY6{7qE6Pxh-ds6H6D;~GW?rF4&&|Wo-`7AT>Vvbw zGG1%*QjE8tI%FYr>FCysRM9-Cz_R~cjS{i%e0@|0*O|CirI9P4$VoH$8ce1LP~_WV zi8QpL4H8iszineitkfdip5RMN*fYY4?Re!W{?m2SR2#L&m zY1*uEQXu@B(O4!#Ga4pQx-B|<&_23WD(MQpK6 ze_*-o5sgJt?_UqIs_YHn$!k<<2Ki-Yv(oft1NvT@{2=T zJf)<2p%aCQ8>xiK{e4;(91sOr|FLu*9T>kO=F!2vgQEwlaR*}M1bunwx{zEzlQgW)TmUHe4ZdKSAI(oH{@7v9id0&GXZE&iqE9hNxRj zWTDNi=9s`2y8qkhUuO>Zp0b}F`cw;pVj{MgB`+W(o&-EX$7|vu9`r7u_9NE0ZM5u2 zj?1eh#l5u41mSTJyuc4vb04k)L|Ff{3I{?~m+_q&CO&mtF5F8AHKMal(SyEqKWyjnC}IM-Zx$j9bV( zZXIqbwW!slcFpU0KjYagqGbsqS${4mg>z2}Q|wdZ>6ka?@rwUfF7FnS-6wE9Fuagm@2omHr!VTm6!!^no>VV7wMa?QjL1Bpt>Y%JsG zz8Hj0<0ebHIs=-oBL80QF!XLs}gM z$A<(hi;NA$#Cx?mX>$tt_wB7l12cmIE-;1MkTZ=)#Ur4(NWrv4Z5dCAC|zM3Y9ESv zeZl-`fvBg#Yw_*f%(@E)caLXlxdOEd$YH||9zGr^-!T-~5?Y(B!-#zOk%Pj_eC*{R zVkgIf%}SRmA{ zVzVB@9y#&C+{bBcz^scBqX=Xq&ojqrXYqReynGB5#7~NtU3op4`P&t8cdDWPS?Z^Q z=1HzoZzPf4^4NuT)RQi0JxOyy2TlouisPSec6f*}Mk3KGQ?VkcfAiwhSKrmj2-ir* zDnYhPhi~r&x=s7Z0vOCw`s&k$YhWni|PpuQf8g%s}jzzQ;4xHhcV*@@%eC37nAuVbCMFa#YM4KKY~OHw=_%w#EK=Lob*TM@c}Kc%aaPU%+xTr%xZl z2zIf$E^91L@BI*20-d}#%KJWKv8rvcVs=1`A z!)HmKhID;yVviBiI>%vSZldJl2?vZ+Be4uBEKODm);Fo{G`a~4JuSbcf1i;De%X*W z@|`3ZsO|T#;5+Ra4H|+$6&2tlhw|Ii6xM5hQ%43u3;hw8+%rVa>zXv^1lr0@x`*2? zC)Au&GnLLi{jUD($`1}ipADz#rsv@xd^ht%s9x&GKt_)f6@?%EAcyXWHS0TIi#KM{ zmF6*><*k5y$huPnPc)^^)t{$}z(T?5>_ge6NYvTL*V@~**Dd*vK#Y9p1&p8d;a6rw z*@&N-IFGg)XLppLMlm}s}S*e5YPk9 z(EZm`-1Rr-sBEn4y;2Kdg4wD6Ha^zTs-GVs`_OJC`7P$2iO~`77 zq#|eABB5s)JW~g8)QVbUO)sa_a_tXg_#&#EAs(`A9j1mPVvRo1_f0FLD_T}IClM0x z1bR)b9B~B@#6XU4#qR`E%5r%&tOHdL9 z9fT=p=o6b2X}IdN*A)Y26b3q+A-f*rsSXt2| z?(@IS!$*vvPCdFI0$=x#7%nLOalp_d-XT{LGI@>io|IW168iYS{gL%ecVnBx)&UZz z9eGxg3SyHeTq*4qHU6bIx*yIK)h2soDFroh39+SWe-H@X;d0#8WK5cL zC>_78optCzecCNh)nxGmUWnp7YVN;PH=$~$SG^IqqH7gkUTtWh|8di+=2ec0WDm3g zsjhet7douO9~-^K8YS)sfdqYLwryqe}}g zvB{SOaI-aC{s4#UE$-j2O2D-LS&`O^$>=n9I z;dVLc=t~$h1SWltH1{diJII9lFzgt-k*+u3?YOC=22JoxsPBNHiKOH2sVGs0U@T6V z{N=%auLB6BN&ffF5Ea%?4EjpkUyN}17&oJSFs8eQJvBe& zTPWryP*Z94RQe z(t*K>kdde+_qA_>zS|f!++;HyYp+rZG}13-)r;jEsGPzxD+82@&%Uu2mb-JT57=z$ zUDkUQ`OgLtol?cB4Km5W8X@j!H=Q^KHnw6;099z@*O0FFeG?Z_{ z5boC|56O|TL%e|D=HfFC&@&_$`@g;$76_U&exPtWnXX3t_zAoZQ97)Jt5IE}e7-1+ zQ4&`wZ1*T21BGM$9p_g@46KE$j;^kXi0`0rE`dY?p6nR=-2G-O_Rv=!dH>;q) z)r6KfXeAT4CwgVBOx7&4Tu##t*bq(!i5#42r4Mc;V1g!9DLgpr+BhS4JA-m;P4@#p zT=6xhI@HyJEg!Fi)4Htnf^2SMFO>g2EX&T@%UJq(82T-sUCT&Tp|YMmd4C2I@`2Ce z;v-s9Sqw=T$gf(My?!U-4M*LIdtgZ#7do%Ew`UL*-ln@QV!_-is3yVlo957`Pb1+- zS!DqxxI|(Km`hhdkE?J#-lKN?gHcn-U5SAubdv$(=P?LY5=w>Fl~<;gj?JQE2ze=P z1Kd9mzsiC269eCyHR5F=dfZJi=9NBN*2g`k!1n3fht(cI6T!o1s=wH-OZ92+RRR0D zYutwBq0IY1xnBPAkQjzkf9W*7+UIl%%(;WLW$X_AI`%1W^(<=i8!^q>EPYPisy?2C zocqE|Q3@A6=w?W2_J7Z#FiXdUubVv!Tc#pBPnH8YDfWGL$4taweTo)0>Mk%0#r0kY z>>lTA7lhwi7Sm@&dQn{@Z^mjqUQCb%MonaZE{SQ)kXf4+CiXaJHs{3c$*JzKS zS^_{~b2}!b+Sj7llgnpRe2|wCyRW{`0Dx{R#|d(+EaVAy`N|3duPZs@Sv7-h)Th^bGc>vT*?rNT*Jfz-b z5x_l9aAZ+pILwUfBr)09Q5o1y#)rh?au1>H7E3sqU^p8Z!{m9xun zYTjfQSc)+HYk<73BITs0Ukw)XfDdsJ^*$~-XX=L(g{ItVOsR_t7@LXW%6M&GB+d2) zkz)=U3TY{My~m&CZu&W1N%;!LSc4#PKv3bm9C^&#JB~!4adWcbBwQ>J_G63s5(+6M z;3y0erO%xB$&ICyhWx>fK|3p`IX-m@Evm(*Y?(B;uIyVuM}Cvn{yZ>ie$!=#bQcSZ zlocx%D4X}Xd3X&GNNnvifUfL6^;$3&m~h+e(E6fcrH~tmuOT_MM_QS&tV4SIqUd86 z*veHT!;w2%{FVZjqkO7caML@1A5YC6_^#c(EfnnnDmS8cf_W!Hw`10a-E!xr1OeAZ z>;Gqs;m5?$kGIBZBC+Knri4hc``^>jx|_GlIC!0LFqj3%)TC-SB?wvKGpQ;r|MKi(tcGu)|oT&J!^(gXudg|Tb^n} zq)3&b%!C%0eAjZ?o;#rYu0iVv^-dexG?EP;4;UX~7?c0O%xV?z@utbAr>h0%S40@=3sW|oFw!5Ro?S?8)i)*2C4)Xri`IPNtC-1#0y8`wxVo^8mX{AJbp z0cM`&a!qluJ&^8Utf>@+g*wokyum@8n@(qPjlOU{pol=v3;lPi9OY%qH(=O+-=!@w zV~32Odd9EAT?2jS89fC7SS<;sTOww}!%0%`YM|1e6>Q9dpFPkBGFxDdrv z_W;ARor#&aQFvh?vI%ed=Nxr7&p1n*7{wRsw~{(Xz|?e7!y?lVLf6=0aVGCbVK(e| zQCGA*@PJ_XKG)_IzB-qSerqQR%=FtN!+d$BVw<#xx~28}9I={4eUc>Tzbbj`!2$6? z=B0zi)5TMkMK(urQLjKs{pgAFXAoVY;>@g#pB@Ld>socZxrET;hGWp}T#p_A72dC# z8sE>K45FY88T{Pm=24no5EYrl&zzxJ1{FvM=snxNUJ(!t4J%@4RP*B_!!>sloC1tq zCa6${F~?%%aCLeL6~Pry&V=RRM7Figmzt(hH+E(@bfW3FxT^a9KS-k8dgT_}vi*2#o5wg8F+E8O9F4ts zD!E%>v!2H1EX_y*Oh`UmcUe1fr36Cw>%0ZVXrOEj{;cP=%gb({(IAp_Yf4K`2Ltj-y zDEJk5gq7`*iXh(jO!N(i$dw<`DImh!^&jPr13E)!s=o%gsgcM*R7#4j8gzed8cQJW z27kEfjWkzVSpHN?|MLOWgw`l>oe+#!@2)FRB|wtrd2 z7qeNejNeI!XpZkiKK$9%&bVaM$#mJhb_qzh^astilea>A7Rt`%7Ks_>eU8&JA|c*T0mJmYwrP9d-t%!i z$Xh&J^`;FPc6OozcD@87IR@?n+RX8&K-e1li1|0(fj|#|FBO)LsWT;FYvK8qk8Ve8R~JEPI5PY@I>s+kES#;G>feZ>aV#twl-kl6RfjX!Y009d zCsXkc^t@hzo$4a`72#BW@{OMYv5pQhk2-yD#eJ6E|9(_8Q}emxDsD<#GSp%M2((ch z8oy;PbzrH{UHMxz+f^2x16nPr6IGN%JF0jKmGX*``bjHYG1HZA536L`gVV-#T#PS~ zAzf9Pe%2&=H6MuWkKigNfnQ00Q;h`6)?#)&z%kQ`>fn0+@BL0nBo4AxqyKaDesZ$Z z@^s$7c9Hv%vX#a&QJ9jfz98R1OYfOo0WvX!P>Uf=0NzQ->GHlDRx<||ezt+TIT2qFs*U~xWoy=ggCrt3^UF@yC6K8Q(rJHpq&xCG+N|jje@p{CO)li+dR21 zj5m~!@aGeL*cyF0rzZUJDc%|+*)mbvLwdzm$@PN-_cZCILZJP~saneR(Yw@0P{M+n zZ{S#dR&Kl91AG|Qwn>9j<0(w5rQd9-C^la(b*$D^%v-iVSksh`b zeeXfOX}o%^l40*5Mi@H;pp|HjeH#xI$r$WBdMvaq^WbqtLsZpGGq;9t@RDB>*+-D$ zqQJWyh2=L}C^}S41_a}Ez{O}i&vie&8^6z$dB35vkGD$s$97BXX+|Ga;@ZU=`L~wA zZ3_hl23pih85U)x%$_P`!E8LpbRmx%vf}~-OMs7OmVPfu_%bq@TFDc50d!YroEnW3 zem(&aBq`&hS&T9G4ZX(EBBq06CI`MgyLi&%>%z)awXBYr%t#xRH-*2F@9Lgv=FLO` zMqe6oi}mw#Y-?F*Y@~~0`jS7gnTDU0ez3QuU=3?`=sZmp<{}xYUjM;9eNDW8xbA?f zR`zW<#zb84So{iLUi>)wzTS$OW?m!*2%n_y?_S8Ci-rYM@aS(?B%O?+bziR6W;;*7 z%2nu(%x(mta$KqmowP_%u{S`?M!eWeEik(xdzM797$3$H<(QW|L@z``Eq%)Lnumq+ zZ9*kC`XAr8ey^4XXFQ3ms{?6ooQX_orjS#N)IhWFtc$ z@SB5`5*lXh> zw4z}jv7TGBr$hj$Ojdi$yBm+=)Jz1#msaWDIl5ktbGY zI0fU3Clyq8M@1N4)}wb7&`VKGF_+ikDlPpIxoratA(QP zYvwwvqb7t73Fhx^a4RMD3+TX(zOC0)x3MP3gR67By*C;tYQYG5=!Fzs^qjl;%14ldjODke7nK z1MMB32-#w_?y0YD-&aLa%r~9njl+r?$YHH2;Gs72J#Z<$1mu4k@AeOvQYBysu_j$s zeRO#JhsPThScikYJr=vk)@uL^uiLhqBWoA@!ccc_FE!cNdLv33V%n0r#YY5r(2+wZ z^WtUG;gkKCFB;G(^<7t`N|N&2Kd($K<{%L~@qK51c3w2UXblyP4P z|3?3Me;%S*L|V_$fK8%o@dpJ#5`DC^v*Ayl4W8e7d6cq|on2*PdQ9PogFBjt83AJ+ z9DEy_okMjcwIBKz!aG8A)Sjxb0fapSQ|)R8&W-0AuDEeb7*Joja|4R>OA+eU{5skh zLR;DsVn?#6b-I&E>P`e4R7Z| z6&-!%qpP3`$wvqPlC*-$}oVR+MOqp`Hyn?Pn*D;B_RmvGR|i1pnq4eihd ziX%n&qPLI`ZeG<~?`;Zo2JN9wRsK}Nz>0oD^#-2Gc`tFxXZ?L1mm^DSqc$S%h*yS7 za3+XJ=&6g}Sv{{}uL=Hc^<}@Gw_&M~dcTS`k9a(0gupVn&}h1$5)mF7dqU zxhC3&brs^~z0DQ$zmndsTc`5DwIFNN7R?}daMfoD3qBxG5S8>Ama>*l>w;f1>iC~~ zApE(e--c%d6kSr-Ri3ftM)Qx%MjbwzB&t8%1X?o0AHQQ_j@eK5h%|YP6va@A)e{DM z!I-oNm0MX>Nvo0uJHP|rgxL+v3y-2>KX&j6^i=4=4__* z{}pkQyQPzQtzR{Q()#dLK+~`wFa1e-$(RjW=WB%8jaGZd9X*QM@Lj^zRFfOu@VMYk zZm-pnJ;cpoSjjxqD*kMdFnsn9TKqk8XKwjwbJAp+ZCpyEyIkoJtT}y^(yD&&nb<3T zXj{O>gcRHfluaD$sKvMDISZ3LW{K!t5RN*$y9f4&H}=a|2ySBDz=dE{OI^2TTt4>J zRDiLd3R-E;c2jpj*mu8DAO8Se3pa2;3>1&4Bp+|rrbFt<~M%ite=av4j-3a<-pCq5pPi; zp{GmlS>B5Q_C!{P0)4)h&4=$Scs?_@g(Jd#PyZ9vSwv^jCU>F++qIlb)HTs)$puM~ zLjoWVuH5dbkuhE2q|^Y;7Z1y8FrzB(=cpEh%h(v*1xz;|k~HnO1)Q;_>wHcOXL1FP z)|Gf+$+2bZ(M7P=-T^*HHcQg9+W~!seS~bGU+^bk!@*k<<52iQ$*)x|MbAf1aWvYO z*p0@9j3+Mh%E+mJFQcBFFC=`Dtw?PweuKA1hY(^Gr>PCn7`e|SV0!Nx(GlO(HCfq2 zDeT6#KzkG~o!)21fhCA#{e|u(g|xEUi=#7NHmK4P9oBd)rIvz4D2$^9K*tzK6MxJ_ z_m``Q708@mH=B*2-^5M7}=W>M?+olb&zW36Mc z8P=SrW9&hB%T0qbv#V{9rqhW}X(Kk)HV~ZXioKPJRGUKg1uj{va!(cp_UaP4oCzlh zf4h<;ucIzcRmh!A1&E@X8(P$?Z8myRJW(jUaC*zehAYtt|3FhLpDxkm)^cg?;j#fk ztoJyCWg6YqE7@Q6iVjf>8wRL1O7+4B*NDdHe>kI9MC9l6AzE(vw&yL3Stq+KODUE zw}$DF(``dIQuf;*c&wj6rzw54LSu%-Y-Isx~B;-!JsJo+MNz!{vs;bnR5(zZop>;rg)cHiQ2EU<1874!qK53269zLP zn_21#-A?(rLd&`{`0xxR3rXC0@UN1!9(YPWN@k+rhqBL!(>2Gt4uaxR%tKP_ zvDDPbeQjNJ?ls8m{e##1yHlPdj~?E#O0qUK179$2-E7gX97D4fmI6(Aop}xPo2$&V zV3DiDj&)5yN|hM^_6**vT3>?TqXf@f0z zgIfuj@*eN^s@4Z&6UOspkO}s{^44&Jbsd^K65LAGSnG_FL6Yd*1rHok4bPDO&EX@* z<8(4xO?z-UI~=ZSiwfWCHp*npZiCAFa_505lYZ;N&+Kp^ikr8RQ+MfH4!mWMq0iPm zcQQh>o%Vh$`Q#GRV(uQ&FChG#zKjJJuf*nrBai!}2wEkmv$QA3xYx817mglBKm(U0 zy`q^Q(2l9{HlnE6Bt#v8r52H9s&pw_nY+}=oj)n=P+#z!y^6F@j3&(hwmQjNIBY=8 z^!ze`GsI&gcsGJQJgeWU+zzoPz@$=kx2A&_KuILg)@@ft2w5Unq$G@Thvw$(ls42ccM8{8IAudt#S%a8GJLdt zbqO!?AOO9eR}v=eZ*gl+malL(3#SNy*N2M%Xsg!C?X@eJBk36EKeMl|z5m^kN}u_j zTs!I!^w^A7RW-f2*d{5<2H@aTX@6ctAHId%jt^ZrO9%|%-1N5^;1bb-dW1Sid#Ku0 zqJGcJSo0!g)Np@G6rG^^kwV4FBsfM*Ak(fg1Q?`yd@1JRgF7#;@T7*|YWwoY& zi95-;WhtUddpS^sP~c-r0ZWYplo6X!bEXGW zlw_&b?i;8ZqoT&4_PiA0zT*;MlmnP^c!3}aNJOI2e<9rcp^ru(iBKrSaGS>u(#zb6 zZ@5H@!Q(ZySJZsSUf(i8NJm<}t2NUkREBX*xz^*F62|Toc2{I6_kv-Na8=OihZ@#b zX`r`=?d4Y225a+2GPmy`g17Z-mpR^6ux&68fGUilMFvwC?6P4YNqEn4`&Mk3y20gI z%>G|al7oNf-{?{64X$ROY&3yz_@Jn=A8Vh4@~H8RDV&$cA7%5qhN0Zl!Wukg@X~2>)>h)8KQKHAiwIPh7*96@}VpXmMYxKaEaF`U}R* zolt3Tg9yN`Z^p}UK0s@gHc(i7|0}>cF@^zF390ta-ujW`a_M#wHN_WCD&*wh&(BsD zGxdD(smraQgNu*8QBJDN+6^ijj4+usc9ZU8Qr(UL(#>TlaKyyQ6OrH0lj|M`tEmNjjuN!^0FiU~4<^*{6Zx()!OZM{p{M52cA4aB?MjOg2Lh*O zMh%y}^<8DtB(KZE0I#gQ94_h^)g84xqb%GjRcz`D3`|;l^$orQCiBB2RS}E;emGqH zs3RG+qw~u-uf3Za4Y`8^}MYD$$k>=pfM za-YqSepbB`TWSN)N{}M)$6Wo{+c!2<Z%kg4*P$vt$Oe>Iy}G6J(%A|bfA^Pz zuE>WR8y3TY>EJ>OD+@ZE|ESB;h1r(v5eU*4jeQONdd}kP%ed zol2RjHTY}6hd?tlVYpjjbn>Jmj^x8*Jio6i6+}E1_Om8%U>xiZ>+=#kn}LI?z9s<4 z0^1%&wI!ZSD-ssMGc#iWA%q1^;NbM%5SF39V*a*k(Ps2AZwh426Lb9o%=Vd?km-8y z3TwBd#pqf=U=ibzvSZWy_B`OY@VG5{yfE?wHjn)`sKFSkstPVz6iHW_vr$|=fzc1q=ltd6YM=A#Y?U4>1ZmiR{@$Ck>>_U^2(?0|L zI9SVVmduWBW;OQ%sZS=}wF2r;8&_yplvS8@wyJd$PK8EK#!q!{y30P4(93Q`VB
CC&<@6dQLis7+hXbjejs4|Q_M{?!B13%E2l-f3?11ubg z^kyx0d&QMZ;Pve?mF6+jnVoH3_N-B+m53DQ@#5K2kJ4gw6mEi3yoak|g$c8}G z>&^!ejpA6*S0^?n-x8ebt2Mvk@WX*c*fy}nOXpZh?IFLWr!{pq5v^|?jIn63t)n4* zUK?>yn0lpDqw-;;)x~`&?<9l@5)%-8XAWcFe(E=qtI)9S$@jOl!qftW%YNQC3rN`8 z{m$#9>?H)9RV8z9q~Eh3y8(e?g2$S?u;?k3BM3yt1K)s`*Hsf7=|7ezCZm}NU2CMf zS&Miudh^&aDmrzvouNI7xcW>2Ft{#SnJ{XrQLWPRb`Q^jRetT&(_q_JSBbh9S~!(3 zX2&ABf;>wI#^`mf3gJY*dfY-zK!I>~>F(l_oD=~TMs>t?v8Cvfp?Ge+akzIAPkw6r zwyTw49W-|N(gC-nFND|nQY?@UIS({TXqOk>WZzkA?6%U(M?j;EK+V6t4pXl zf0nkfHvYt_Gx+Ta$B`eLVJeJA zRqcuG9@v~9wc2J!#$jw)Q`a+y#T>fmc}sVs&Yh6wkZ@Ny(COBEq;q8;4|(n>DLjF7 zbq41&o4uK8B4?7<7-UR;Io*x6kz11SMO}CgFxEg1f8IMVZ&$Fo&a}T1XW!M1;8F`R z7LG(#lR1(|AonAFprQv{H*TZ0c zzu6ntqLvAbLFEQZIBZk%oKJbp_H~vO8 zyb6Pf88TC^Z)wrJPLDN=G=NO;M%&;c$B{ERArj?aCGb|)QnM~-m=vHj$ExO)w*7W2 z)rJ)E3tEGYD7lX2L#~;O3WG6jkIZO_4b=GGpOWlx9_Dh;d7OD%{xJ~VNpvpBdG!F7 zC49o4wjAr{)Y0y3lAPZ4Qifyqz0R-{B@Ggiwnf|YDax@r0q=#T#O>@Qg?CiVja!&<3d&1=cA!VRyFym z)U%Iq{o_`=g}<si;R~E+Jwmv9v?I z4$%X>Y4m%S%0N>$Enu%gLwNl!uY2NydgPW!WjaNkC%gOmn;9?aY%f-<6TXZ~&Xqd| z>N1;6g5A}U8&}JdDoecprz{;wq%Bfcjqt(=BZdDfJK>_poEWrhBW(Aw-UiTe*JAbTitkm@AEYR`y z5XG(*k%omp+^Bb7j|XQa>Pqft6LQD*DHD**dU0Y+lMO9Je4=a;la@>cH!>4L!P&()e8iOnRpPQQKX73CrE?~|XTA0y zM^LsAvU2}vO9yVvVbDb=D0k2+GzkYs2lTOtpsT*1xBJnPeK+MPr3^|Lqro%9;!-5% g-IgU!iyl*I-0J|}HNZaTIRTIG^(ejS6=eL7d1ooEHUIzs diff --git a/secrets/ssh_docs_deploy.encrypted b/secrets/ssh_docs_deploy.encrypted deleted file mode 100644 index 8b7e2ffe279402c74a57005d327220a4806500d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1856 zcmV-G2fz4JVQh3|WM5x6_BQu$(kQ%->4|0fa6`t(&xPEIE=okh>cTg)MnE6hWl4%W z%<3W-{MyzND_s>N=jAkwv=;0N%hTRP{Cvd@J_VNs`*gk&F>exm{qkM5tsA}DTlf{R zE3wfuA1RqY3nBxTkXkcD(Homn*7scj70QL=N#-$yyBFcv>M0p_5i7lvUA=JQhG3Z+2M#8S>T*R*KB;@y_|LL#USeKun zkfFh$9|tFr_kr7O0Ql?;<^&yF6V>Ef_^4H5{F9V5G3qwi<-e|_aeeqH6Owo8;Rrxo zER^|Tk@}rbeW>kdJH7Id@zC2jnZu)5rJL0_p-$QD*LUAw?J(|;mrWh`aEL}&i${cZ zY)9c(4&4g(YI$*P$Uy@y4*%%H4eDuo_PIMyknL`x$yy}5&8_!DWuxhtrLErV$9Yk!bromBYjTI}xQ6_n+A$HgyXBLV)@ybZI~VFDh6B2@^*>Lj*b-pbw{G=x0sV~412 zD^63?S_Jp7Yw_f9vE6|`Rzeh$kOKcHQ#g#$nc5-bokDPt0>54uD*tsi`jrC?C6Hnp zyta39hntcy$WBN?9Lzx69D+`$2(Ofh)~8PNRvU@V*T`0o;K)roVPfVBwCkalL{maZ5qqQLtJG; zPN^s$@56cDc1#rP?@^P~J=N7j6JB`kMqKKE%1zt~u}KiyTt{NsNUL5c$j^c_AHyOl ziX5j1M3+!k&H%g}NZ9n}!wQ^c!`@wtk70?|6IWO^3+RzA&t;XE@Qgeo*f|$?HNc#v zroCb}MgdxKJ1&nVgVk(ADAtVfMA*sdWr5li%*zneS9atN$?KHX6#h92?9s;f57^ya zwO}Q*b*P=ao`JFwLr9tH*%M&;EB`lZ)XvM)%znRP_lHz~^Y^V4r$)=d@Q8q5N;|riRzw4GteZ)2amD zu<@cbLZp~&>IO(=XVO6c62;Kj*;x)plTO2Q!68R*?$j|bnm_Eoh%i^v_6mn*HF7a1 zZv_!pari;`$#s97Z*XA>vg~`325A;w0W0Vw{1h%&{fKK1oM`Eeg^I~}WtpZ^ch{yd z%lS?c8IqGcJCDb!LI2wP9R_NG@}l;~Tj&xjC}98gTC|vs2iXz^DkT-j6{m~aFz`xf zH_+Cd`skfz8WSz>I<$zoRitWpm@@!zcJ$K3rn~Y0ebJLgz)Un_68_D1Lk7K)o=w}v zm`MzC4hhfT2d8758RTL=q+sKAxexZ1n7v-c4%Bw=c zvfVjEpgF=yzWJ;HC;fqYJg&3car(huXIu zTy-=%EaXCo$MpXLIYe&JpuG9O*g=d9-maj{#}IEyzeRcUguycE(_agoB<=P>B46VH za)c3lWQrd2sWAp6c;OeEjOnGInOh?7zR#b7V%4x0$sB#%9_K{@9Y6rJ+^iAg+yv4` z7~bPZGm>{d1NhK}DMKyfLwaxAH z;(X}Vd;1P}+WzJ5u1k`KR6bB$I$}XI5B6xx%}Z|)Z4mFEyU3=${b) z-YJO2(&)IoTz6MidbNwn3Mv#8(69wQmsj6Y1I3Svw5o+r(*XNBO_<%$mmh?vjKI1ze;=FMc9)PSS&U0!++8Co}z4{($e zWfbj0!zl{njIk$UDNDGhSLFu*Fg|WQBX7T2=Dm3rb)_pHap(c z#uy!CCF_E@SXl}=qRW)-Ry?&3(eguCOXBs<+A$*!`*IeTRAh!q zk=^?cwg@7NIYoHPIL)lJ2QB=k%R@*l^2dDy)er*Ua39)BLeWW7A#&D)Z|vLX6r^(d zQhsCMs0=WMfC6)=6EH|PE^cl;y=B;BSh1K14YUc|iWq5A)6_2oe&xUSR_|ZV(`@{q z%@O^n7UW_lOokj2Yu2~Bk8Pfyzvy(mFuc=pm4-Fecvm&6P{10@vbvmPkgD?=iFO^8 zo&a&?Ef|l{deDDM-q4m@QCkN)aXJMh7ZNt{(B>F{YYM5!^tTi(_+wB zo-fHKX*wLCE1GG&bRXve;*87a6D6SWX~12m9ploF&tg;TBYWV<#8Z}Cl-+gD{PZA} z=ty0A7*`#phbs-wEL>NUQ9f|Wof8X)kNn;bS%e^z_B!Sl;ffAf$rd^=b+(&cf=bUA zs)hsd_YD-5Kvc9lv?bADCs_g82BkmvLSV{HD+jtG@9#Kzg?N4;I*Hn>F^9iXZqGhY zoEI8{afDS;VoZUzfk6<_>MN!RwE15Ovnr-iIk9!3$uZEd=Tv@+6>u=DKxq5mJS^YB z{8M01FcTi>`y=9q>j;LWSXRu!80H7kMVDtS(f{W)Kfj`nCtEtL43v5nkZKH{e6@F5 zzA+h@3e@BI3@EaFMHTe}!!s3LiU=XsUGL0B42sz{%2BK1<-5>TlX8ov1lgGgG}njU zt5t^Zu|QvuD9I)HbDTTU<1j6;*#4NP(Nk}aKHD0eQY}KR-o68fQZgVcQu@i$#aqD2 ztP|SPD4c>tyQpwTVBNnlP`c@CE8i1wFCqaoUvAzVUf%Am%!e)yQ(QE)QUyTBNFN@p z1L>5XWw_nBqWl0$^Ghj|KZ?!(IYPGssxNSt$<|41*k8Knj^i{$N(^4bUw-}n{) zbvoiZwUmh)52R$24nNXtw^y1K|1np8pv{`SxdE6e%x`8Mz#|@qw3qK%)0qc|PTWH- z^nmdltr~vkI_ZAg3RiD>bgTpli2Ht{h&wC8xK4WXqs+? zQuo*(^VNdZ^ys2_PdTvwDD+H*hF;0=h-)Y|n@%{e%tgEg3iB(Jgo16-$YX$Ehoc~E zU?(DMs3=;h^v#}g^ljo(f9KeAnC~$eN0gft-a+x?a^E@k?&*96D(5XxKD*84j>n$A z@rOII)SEN`>tS4^kRbZ|xBNjEMh0r_zat9T=i{a}i}X!bgFxYy4GU6QXITL*_2`0x zU+WmsI(HgP0(DP-JpEPWNv-*{f~xZ^0LI8{$s>H0{C(U@R!znHFNs-pW+t}*&1vJ~ zETA{jg>o4WiRA8Dg}01)_5o-?O>XI|QLuQHV~ayoLq?%+J>p)~UY0Yoh53}|cE`kD zX`Y-1)ns4n^Q{-?F%M+0Xm-g~R7Sdnv1Bm8Mn4k6**>-{Mhh>a?J2XAU@*Z+%@Nzj zUv{NzfGeg_Y)Tu|TuRhr9n8Zko1UwM0Q?WoVHk#BW<6>bFp#~1bx&tFIS{>f*t{ST z4NfD#eE(2w{6uiA>wNQ)pC}_L8EU$19V2SA63RNW9zr~xclFV}a_fhTV>!@P&Yvx2 zSTKOn)Ao_z6aaE?7*6ncYZUmEDkr6D43G4fH9=qs>qgVA`yhdU;3@xOQ4FKjvkdV=vdd(l40lc!eW1UhlW~m z3jGrSamT4~#7Cp4Tc-687b3SvFA>2|tlyNPX8`4TLf!p?yU`h88SZ4P$xODg!fsy1 z+(0~Fs>lW~aUAE=X}b zUsQIt^cD1tNwXV@D=!-b(=q+jf%*q=garvy_ve&TU54WyXuS1HPMKC}SQg^9-xBbC zAg4}IM?%*PYjI?!D}AaMTwqyV%|>Gz_27Nb$5#R#&=WfvNnS6P|ewSITNGnQDZe(fW#(hRC4eBqh&91;kHE99}j7M2GH* zsOpZbSG9xUdfQHrSwXdt{oeB;W3`K19DxIoRtO%JmGn;S@WLo26u+ zbS5MA`ERNm7|YxVCC2FTK8ax9`4mI$0jAq0*_z0r=FSa*WM zzdqS<`Z$nF7jnB>WXrGFXMDmm#dRH|X_7Kdbx=#G<(odw`dl|p@ubd*AHriVHDXO! zPWb%qKX?pF!oN#$*Zp!vfYzLd@l*ksX~Y_7LJ=;pNGIWS*MYr%^Rqqx=}=n}1+zOu z?0Y`}?o%r5A~YF3499MO;~ePQpJE9JHwemgJDk%;$yIB3L{f9GJS47+<$t=;)Lh~7 z_7iVtrRMg11ju`?7i^O-gRb2ohI5i|*wPC>9nk=pe{Ms#9*jKeS7Lv9q(tvZ+kVkE-gyYWxO!})XgArV*Km#BD1NGruKbJ>LL8OO zLbUF4ZU+kgu=XwQy&m(z|00q`f+I(LsA|J~{c!)uK^^CL0w3rzg%0+AxtRydHfQ7* zn`G>jsi5uE4e3mgI+&5m-GHF*gy#G!P^1-5_9`*REJ~fh0{su9Rcc5Z!J|rc_V@pP z8P`x9*~;MGsp(h(PaGFM=?#LFSH4>$6>+m-nSJ3&Yb&x4QU}DG;30U$M^fXsjm&!C z(@C?dnanRdmh^Ox;;*!-)r)@`M5KiR9jc@wBFYDPaP}?})#T~k3~-Y$`23CX{e?qF zRk*N9U8JS9+Jm)FjOsRf=V~$;Guqls#})3487}Y`fJuY9sE2c8{QxlY4JD$C1O@k! z-3nRN57Tr}X;+*jD3*tjhK#!pewOo*pDDXL|_lj~WElh{|ob5X4k2p2Xj zx^H?P?^IBSVuY)|UK9N!UHH36V7uG&d3nnG^k(kn;4G`E@k{9gJA(4Ha_&g=Gw;E%^T6_lJ8efFs|pQ%3l&csSTK9EQzCD4Ot+XIN4of>B^J&mHv z7QZsq&3*KM#FkwTA~Jp)-`Kq5O8&|qm#Ne?0{B$wdbrCj!@*yx{2svZc)(Zmw5mFI z0n#=iy!n3*9>6UGx4^XH-D_cY>M5wX^|l7DS4(%=PK29C$kqu(lG?aI2u1okK)n_q z?crw&A@5#D3q%rq1q+!w`xvNl-H}3>hoxGYOX`oolR}1;2~c%-7Ar%{2dE2LFys>i z7A!Aow=vfH3qSc+Gvon_vFbd03)fet#9-_bHmm#8`u*f#`7#EoV9z(#zw{o$FovzN zMOUExjJryaz9GYjwHC%P73F(%wQ=YVfHM1D#$6=zLFucM z2h294b|4u)G_eCS zg^NO2eomv&T3$f)6S%ohqV>d4vEw@a3|a8zp@q}l4!^d$NG8^sh8fXj3T#J9tYUh| zIseEasu8Z^1WRXdRQDvD@^5DR8k+*@4G zq^Ev9r^1vqa_c&)#^s1hj`vJH6|KIRV5#23J$*cThV-blju7D7US02y2sa~@Ago7V zkb*pO8rqQ@%%E9)l&C|#4V2;ld2$zRc>kO#;7?*H38Bs7d}#YFjIr=IrM-)k`fab| zLnh5R#$#MqsH0lt4;}FK?QBMh8DOx9gnG=ZfvW%I zv!*)1(xE3Cr^De%1e6@{$mS%ZPRl>E7Q@y~3d*GPEC)iJYh7<>>quUCZr6*KR6%9V z2;E3*{ZJdB!RuY&`FR&9$y5ioAW1Erx+%OOA(x9&Ak37b>T1;K*8&ye+wgOd7oJc2 z10c7PmIVp_|kX+s&QL zB4Bt8yh*Y%9&{xRhI*Zp2Wm-ZkDiQl>li$~93Fl1L>$c4A^3wh_Pm4Rl6^Hs_H;u^ z`EhV60KO#d5CIT1t+y&AmBklM21|x7G0u74nQ!?)i!c6^g8YW~_fl)9hP_1D?_f1U;5ipNugRtjL#Wfjqxx0PsjXv_uHXgHt3{eKBkNIIx6_NLSV{hFw)utH%^)1|fbnK#2S9NjSG;ny<$eDk{hTJ?Q+Q^xAAy(u)= zHpa!;HbYJ}5eD}{RttWt4d%$Hs7&8?@$u&H7nhEpcWvgsI;vd5T9l~f;Y_6APd&xL z`C44w25eg;mfVEI7Mjr3|``JHM{4{PTV)@ykt#l*M=r<_b{TJ#LKlu##ezwRoa%(bjG^yWoWM z7E2O>#r8+=5yVz%px|kw-YS^<`vBWvLJ?K)IjA@cB>Zi=(_scytJu7uKF<%052T6h z*QzKum{(P?^-kX=m*!)N#GZu)7cB=HtaucB(Vm#1T#rI_OSY!y>Pgnj#<6Q6WVXE4$r=ORo-yrn_H<3+Us2bdG`=4??Sh+j zGj5sx!;60G9}%YLEw3>Sz&9F$AxphgYaZY@EY^=+3kyR@O?$Cb|{*c!guwN-OF>wHszL0_#==G!u_9}F~kOp`1(gxto)#2d#i%I3qu zCu|pNxx!#_Lju2#CzE#TeItB$!Q%2@Pom4QGUcBQeu|Dox%0HKxu`35tz6-z#D7{gTJ2Saz7{2npJh^ByaFaFP6@D07qBBhd*E|2YS`tdK z^IE8~Ob)k_9!FaS=Qeh{_+zm|(@3{hpO>ec%3~iG$sI|sK*Q8&TvsF|@-{+reAaJ2 zCyvD8H5M*jf}FFJg|FQO2P_o~2TVTMV_is3BHnQ}wdaUPyOsaCuqTs8l&V+|F$X{{aN3*tMHeUUqZ6c zGEbos4~ue^s}DUV?VJ<#4(o0v3Fs55(oNb z7bd1OnZ_*9cK}S=N`NL`iRMC0;@hVPzy@3K>sAgquaTAJS#iD~=FP%XZ2ya-Q2|i= z{mZoIZn{`umm-k@s35M}Bj7Wwpzk8Qy&hcsJ4YA@VkHRx(j9oaf5A9F=WgEYV$+Hp z^%efNqrwt0E*K6w=x)g+v|7J;^HVQy*Da`6rzcOtFX9zu_X=(SNC~OfuwcJPs!=<- zC?SG=$cO}DG%hHL{0G+j8YN+Y$PKD0qxF*$rro+>*KHEzS z^g&5FvnXfsB-ST~u1t&avjD|kbhH}X-TU41V%Bj*6)8ao)u=#TmM}SS8ubiasU+69 zSl3EzK*l3U@jz0KQw8127Kg|-6Xdf92`In#k z(732fG|QH(?I(NlZ$Swmkg8FvCq?Hmh!oo@-zlF>K051LkUHmoCuo-Q-~?zQDh7|= zf)#Xs?sleCi+}gd-nf_NW(CD zNz&G|vnZoK&suyed5PrfRt`v?w0tS(GPX#A=PcJf--)lY`_&l`4lx;?(Z2lA~;{t%_QdIbNWa`DDwQVQO6gpnE|ONKY)QfZc_ z7mhmD#>rFJ%0P6~wWY`{9mCgzb+=Bi?6u_f`nrF)G3@q`g40~awJ31&_W-oOM)L+t z*}hT_S9iVdXIGddAbeh(2r$8H%XDx%GR&c8a%G2rkoy_sQ90vleXBkpZ$$COT+(57 z&CQzNW`^h#eN|zPC_jIGVMN@lF!RCRFc>&Po2ASZzZrm)+>+ojNP|aQ#I_CoZ0`O1 z86HjZ$E}#h%+nXI79CHG?#h4as0`$b%e94%aNNuQgSjeuN zkw+>i=#|AA#|sdq;&~izlLl)D{}f8w`rI0P{;1{EdjSCcDNn+bNQN%LLDYdu zZE=L8mN{%5l+qq;H90>4nSu23e&leXs@fNy7R!X;-GXpkb5<@_Ffm=hOk{px%xvMt zV~C<)jKQVqi4*Ss#AmAAbfvmvC3a+iR&@2l;rpghphWJfXg|_gRuoeg4M)-xy!2My z#!IZEK4J{T*Ad7>eawD{jU33Ss%zJw%N%hNJreewZo}J{{khQ1Wo4dffS%y0LWRT5 z)p$jt-s-*y5fDWH{(MR%^lvdD_kgoa4x}fK76v;3%IA#?x6%8B4S@fd=}}}rTjj|H zTH8^Q@j~aGn@)O;akR?1T^W#%15Tg90fR7UwMVd=D4h`&?P9^`$9Cy*g+BwO@uiWd zu2GT_dp`|mCd+qUkxPw1b-%M8JNAWz2}w1j#>ID|-$V1A@AXbU2;3Yd8Twdd3#0U> zyG`ura62y&SoDdD;57~dGtceA5rL9VZ(&dTmF(jnmb)_M{=&UJ^t{C8WtmaR(4Wbe z6L%&|7bKXkhYOb?3qNXh@035o^<6ek#`FfxwP=hJU}(qMsgC48lotZMosOaI02@F) z_ZjpR%hiy7|5LmYfm#ZV%eMD>#HGvXXhcGpBNlTA7f~c5*=@Uj4#YVZ!fXBG5iDU+w9jT>u!%Wyh!JG_?)X%Z<&K@XYtSdBwL z5R>7?W%`Gn5IzOb_QOCqUR!>@^hP=5O^pn*LG&RL(WW#;jFZ$c(T)@D@vvqpqknH= zB)p9tLSkaT8(Y6*GuCC&0h8kGv)n>C8ElF(u%#4iTXfczNoocs%w!Xcgb6hE%Em z{@R!i?NZd$F%O}ljvsfoLEE8pGmfq`r=RHsh-`_W9;xtefM5HdfFu1mKFz2-8EOLe z@qb6tZ2(s7HlIfoCpR&T38VaSrW?`zV*So@*S%n%*(D68M)nnGrI?hDK9i!h%(o-6 zirR4d0rV{lWX?}0C z!^1_&qHft~xSqwxcabCdP;S|J$ z7u)oAnu8-!b>T)bJp5+|4BwEw9)k|_2!|bS&X~0X;y4hirtlaow@06a_vh|;)*c)Q zv{-yrNJd9u9^C%Shk0U7m{RcvW)|a=x+f8n_lsHH_&t}o84Bc<0G-M^XkgTw4GbhL zvd>kfs+@A)EUy6x7oITmt5YN(GR9x;s;7pb3#chI7`81`Afh4aSbjRFzm!?doPo2_ z6lKKRW~d`XmD0nU&*x(0MbvKsn4Fn(T1i6+<5+Q!fP4&|_tUZGVdO>C^9mRmbu+Ej zb8u&jR`xWU|2z+~J&OtT^AmPL8fUL2Idq=jmInBt^eA!_^+kW6e|4~eNHd4WK+hb( z@T+?RfuG_+=Ms3qdcBKLX)<}o(;^JSmzJ>W8}S>m`zx7!^IjN_gg5gS`aukvqa-+S z1iTCM-0l4z3IxQMcl_F$tm6)icF7AD0k$-~n zmk|)9(s+ckvxOl`6=dLpAPn}C7J-nLvPC%~(gf~&+CK=Z1qO{9Js&=GTSq>&|14Mf zy83P~qTJDF=hWY1nJLTGAhd{e1;jjf+07GiTmGwUJS(#Xh^c%GFxNgN8%Y&{wwh?n zOPAbNraVOtN`j{oJ4!VP8SrHMZ}G&n8vUj;?kA$-Jg zGd}v>m4}tq|LNzt9ujy=p2}>1g;S!4a)gK;(k|>$F=@IsT4E8>rqt#n!orH1W*ura zH(UW%bl?{vq9s?#i2i`ou>xDvqEWI=qXCs|kJYw$cBXfjfy!*)cHPZCI5TEyi=E;( zo*n;d%8p-s0;nPH4s}<$BPGoDDDze39YSRNA*l-`L*=HlY&#SDQ#3Co?@Dr0(_VM_ zqxOG8{+)_%`D|g^W_ufiO>)d!qMW~&Mz>V3pLZEXW=Y=gv^(XKw@(6I9G!|io1&r) zRD~1+--ysJ&ofZ$gXuBajc0Qrli7?Du6N@aL;2&uO+F<=Jcr=v_Cd` z`RAQuxtbUU@R4x_t&6;#sINiUvR0X5Uu2FDQaQhVi*{XkxarotYUvCXGU73tlEz{! zUSI_5lrLgoWZmCfI4CX&NZ?YU-*wYQtJioxe9&}f3gO@Po!;lRRp;N@<*N^Zm^l0` z>MX9O3HCGb1~p{fGUYq~we;4foO;cKF^Le_i_4_coTgjah8L^7lj~cYf>Weynr^<9 z_oH&Ym$3&{#U>58Q$9DQ1=>+*GnkX(o;_b^GGQ^8>-m?0ND}G-`HXOH zh41`3VcOF~I-lbSgOR?+<#(WD#+mUQr}egrXVx}m$LwzR#B8l>$nV_|8xtMucxGKZ;Q57)VqDi$EhUx4bsC3ULPGqTaoOuW0Q3IcB{6T53EP-gl{%pPI4ab9ai z?aTn)QvGTq3y|i7>Lc3ANamw}F?0@`1vG#~Mz`jSYjNZ6PIFyEVz{{s!=R$7M8R{- zZXq#_`^+U)f=3GW%)kA`g^N!-aJ%_ZPZYAibi(ua0#WVS$0bUz48;RpoS-DP+=XTp z?$}pxPsv9RDPV8H2g+8N5s%xykv#9pGUgfqgJ%%T>U!m)K$c>EmbIKAzMg>v2?z4P zQP;ogT?~Y(ySUUD?_rSr7;dBsk_2xv+xrc0E=S9qSBS0=Z1!pn?m*h*8_WCyU+PYF zf2~hc!eKd^-gGW9P|g{0RAr$3U-&w6o@$WiR_0hDa zrOFC4L^*tB(RA6JOp;ae*pad^24=~|1V-2NrsBGRXgBZ-6G4e3Eh4wr;<^D=8FKWB zYiA^Jspxi*P?om=)k8Sjym!QO9}d7!K8^=QK)*f5$14@9U9 z!=+>(DCS&Uh5}!#sE^d)q>ky->EVY|yd*|Gc;8+$Pyq)R;ta@Ly?XYRLY%eSB zF?X2)2pxp-w*YwwR&xGhhUcY%;a!HZ`i<$OBft9MlpjPmUf8JQogE`u@&qYI6+;{A zJYdu3|GED6d196uN#7Dze0i7yu;RkUZq1Jf0IfqnUI-HMh;%8!)BOucK}jtIY3bH$ zF7ar)-?)2jR4{>4L^o4C$^-jJ3H;;+wi&YZ-!G^338^Lv;8nqY1MHbMqpQMUoNql= zIX{rxBac>Z+=7I>UkxuEzq+)M1%FcBwb`jww~k^^8R-i1Vv-GjTb_0~10g zNx>XUBlG2wCi+AX0tL9=m_ZuNSm8P|?I7oDuqr%PhaX`NevvQq-7+ncRolJb?PYXf zU|u>fcI5u1MtWda?$rE@k4HGj-G^Tcs-&HW5N+VCO5Ck~IZGAOiC=g20Wpn zeUH~+jBjb7y^K@L?aQDCAA)a!ia-<(QjqUeeqfj>r%FyA*rQnrTT=H z*=PDwvsjUHaK_0z$WNY)X!0f)HsAuzbA>X*p=b*c^F1OC7vzfYSb*(`+Tx^DYkhR_ z!3poN*&YXQ__=;K-H{{!CyrP4{08;ezaPjK*r=so&MwhIaVm$CCHwHR@Qq(Fsm?ks zls0z-%^`OXxa|$6DG4{Om(4i{k^=?#v^0ZG3Qz){e{jbo*1CmpptbzN4yGg_L7IOK z8fpI91O)Z;6UtudP&LK}SP!*i`M zPkJQ?pDgSz3+bWIcQtgm7t5El<`0S@HlYK7x^1I(?3zXDkkN{}3+;<|*$#%~yg=2> zyJB+{uOA2$AL^vTV}MkUx_ns<&*c>3<$WT-?_ecP0IIHH@f8ihux zwmO;s=zv>ePICtnu;NPPLxFc@DH>)QzTC@|Z9HcgGg3zjEg)alYPt0l{o4{JMc}l6 z88p}RyYAv$P}ovtsvH2EQRk_{sQ&gpS)ba%4Gu6;0T=QyWZp%!)ckB%H(WoK8W z0H`1sB)wxz^mb1SPKRSq0qXmy$h{$%XM@raYacjb0=YDUp znnf#+wv|;}w%mDqzeU2%UKtfRu8t~|dKaX0+?-aoiAODqrkb91_*em?= zdsfFI>;m4YHrF+5)V)HZ@~g(Dy80Z7kLInI&EcfqA%0hvM*iPJaRsIIs2R>Ah~QWB z?mPX_a2l%R2MJk*>6VRER{#TLLldjyxld}t$c;hHdQ1OEwhoDSv8|d6RTkltw}pfa z1Mp=#sS9iXsG!i%AwQc$>x?gL4*Xu68>`cI&@|8GkO$C-Hes;l{Cza=*>@r+HT%ol zCzq#_r}=84ZZ|*#6eOGP6oXWP78SCTYy&H9+{R{hRhI%k$VMMF(ST$@|B|9S{z|M( zJh8v1{FWsEgAdCe}* zRHw7qja4Z^kt6B?D+kV|i<%(1X_sDkS+Vm@w2A{zK5UwPcU!$IvdXlOMsFIN=40{yg zl~~ez_zDtQO4o6Unz)x*q(Bfs6l3P9#pL37C0G%L1P(6qD)il9F$HKUx&TEU;r3Ip zTJ6?Q=5vRoEB{-3v1w1BwW))UG!Ce;^NVX-Ug+?G=eM)T2*f3_t$j#cJ3w`bTSLaSNv|+q-0*dY$U;TN5b(`O zM$M?bP~$*xzIy&%x8}Nw) zhcy0hYw^J`3=r^COE*yRrEb2)`tTfPAUrAFMc>Q(?||_$n+p3)UkSh};I@a_t3o8@ zKX`cvh0d4_0a+T=xVw-B84y7&$F{?h2M6&pC^4sSQPku+OWu83eDp*(2CgG6s3T4D zX0?9jvA`4o!}49py88zyZG^UE#?IVf|^KmZx~$ zEXL#n|5~z%uxNdXOwwqwT-AaBAV1pglHTe!19+uIs0=H<;#}Es4{z>bTXDt?POxxP zX?!hGK;4NzVu(9ZyBn}>PkJ#PQFwIg95C|fw%Ax<@hjfctKkLQx(`?}b(PzG)r^E& zQp9l)cQ>-L43_W-@~Z);;X&?PL%zkYD-zuS<)iqmbKm0VKKO2`BjV##KQcsOij?*o zxRzD0`E{4`@Y8oTc z1h&PEKBQ7(u82S+P~VUAF6_|hz@RD36k@FuDje>>J{s;)4MQD=GWM)}g4o8kfP`w3 z#0dZ8F>ONbLXv^&0_Ce6KWbm_nKOVv_EK`0v=>8gNnbRTnfH7hoV0tBm1Gj_qjQo~ z2ZuAraAyr_OGdAl!h<9V{Lvw!yU^&6&!pqwFoY%nZEL`nviwrBMtDoFta!8xjEAec z(b&OHo8{ZGkS?v-M;+_@TAr|Gfc=1(^)83SoOE+GOS@hB=ux_UTM1x`1q&7lCwj_{ zfeL(~x%epwsa8n;*>_!^s9q0O+JKF>{=xFCk;|?IuoMWab%pDTF1PoQ`7wu(OuUyV z$p+0McJC2UnJGF@$<}}{+26+vP@K(n2XUErpa5%anW`i9S~ipCscpc^4JVY70=lq_ zDkUp(Y?7mc=+L7`XYq>V0gY=j`fJ}S@Ut3biprWqEf8R@I}E}n3ePY^&2 exit 1 fi @@ -54,7 +54,13 @@ while read -r line; do enforce_sanitized "${line}" # Convert line to array - readarray -d ' ' -t line_array <<<"${line}" + { + set -f + # shellcheck disable=SC2207 + # `read -a` ignores quoted strings when splitting and we are controlling word + # splitting via IFS and `set -f` as ShellCheck recommends + IFS=$'\n' line_array=( $(xargs -n1 <<<"${line}") ) + } # Panic if we don't have the same number of items as our column names: if [[ "${#line_array[@]}" != "${#var_names[@]}" ]]; then diff --git a/utilities/aws_config b/utilities/aws_config deleted file mode 100644 index fd9c55742..000000000 --- a/utilities/aws_config +++ /dev/null @@ -1,4 +0,0 @@ -[profile s3copy] -credential_source=Environment -s3= - multipart_threshold=5GB diff --git a/utilities/aws_oidc.sh b/utilities/aws_oidc.sh new file mode 100755 index 000000000..f640f8a4a --- /dev/null +++ b/utilities/aws_oidc.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Obtain AWS credentials for this Buildkite job via OIDC web identity +# federation. No static AWS secrets are involved: the job's identity +# (pipeline, ref, commit, step) is attested by Buildkite and verified by +# AWS IAM, which scopes what the job may do (see ops/ in this repo). +# +# Source this script with the desired role: +# +# source .buildkite/utilities/aws_oidc.sh stage # untrusted: write-once to own pipeline's staging bucket, / path +# source .buildkite/utilities/aws_oidc.sh publish # trusted: sign + promote to final (publish pipeline only) +# source .buildkite/utilities/aws_oidc.sh docs-deploy # trusted: docs deploy SSH signing via KMS +# source .buildkite/utilities/aws_oidc.sh tokens # CI telemetry tokens from SSM (julia-ci only) +# +# The untrusted roles exist once per build pipeline (julia-oidc-stage-pr / +# julia-oidc-stage-ci, ...): `stage` resolves to this pipeline's role. +# `tokens` resolves to julia-oidc-tokens-ci and is refused on pull request +# builds -- a PR runs attacker-controlled code inside the job, so PR +# builds get no bearer tokens at all. +# +# The `publish` and `docs-deploy` roles are only assumable from the +# `julia-publish` pipeline (PR builds disabled there); callers should also +# run verify_trusted_commit.sh first as defense in depth. +# +# It exports AWS_WEB_IDENTITY_TOKEN_FILE / AWS_ROLE_ARN, which every AWS +# SDK and the AWS CLI use to assume the role automatically (and refresh +# as needed). The token carries only the AWS session tags that the selected +# role's trust and permission policies match against. +# (Sourced file: deliberately no `set -euo pipefail` here -- shell options +# set in a sourced file leak into the calling script; strict mode belongs +# to the entrypoints.) + +# AWS account that holds the Julia CI roles (not a secret). +JULIA_CI_AWS_ACCOUNT_ID="${JULIA_CI_AWS_ACCOUNT_ID:-873569884612}" +JULIA_CI_AWS_REGION="${JULIA_CI_AWS_REGION:-us-east-1}" + +_OIDC_ROLE_SUFFIX="${1:?usage: source aws_oidc.sh }" + +# The trusted roles must only ever be requested from the dedicated publish +# pipeline. The IAM trust policy already enforces this (it only trusts the +# julia-publish* slug), but refuse early here too so a misconfiguration +# surfaces loudly rather than as a confusing AccessDenied. Pull request +# builds never run in a publish pipeline. +# publish-test resolves to the throwaway, non-production test role +# (julia-oidc-publish-test): it can only sign with the *-test KMS keys and +# read/write the test bucket. Like the production trusted roles it must come +# from a *publish* pipeline slug, but -- being harmless -- it is NOT refused on +# pull-request builds, so the test publish flow can be exercised from anywhere. +case "${_OIDC_ROLE_SUFFIX}" in + publish|docs-deploy|publish-test) + if [[ "${BUILDKITE_PIPELINE_SLUG:-}" != *publish* ]]; then + echo "ERROR: ${_OIDC_ROLE_SUFFIX} role requested from non-publish pipeline '${BUILDKITE_PIPELINE_SLUG:-}'" >&2 + return 1 2>/dev/null || exit 1 + fi + ;;& + publish|docs-deploy) + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + echo "ERROR: ${_OIDC_ROLE_SUFFIX} role must not be requested on a pull request build" >&2 + return 1 2>/dev/null || exit 1 + fi + ;; +esac + +# The untrusted roles exist once per build pipeline; resolve to ours. +case "${_OIDC_ROLE_SUFFIX}" in + stage) + if [[ "${BUILDKITE_PIPELINE_SLUG:-}" == "julia-pr" ]]; then + _OIDC_ROLE_SUFFIX="stage-pr" + elif [[ "${BUILDKITE_PIPELINE_SLUG:-}" == julia-buildkite* ]]; then + # The julia-buildkite repository's own self-test CI. + _OIDC_ROLE_SUFFIX="stage-buildkite" + else + _OIDC_ROLE_SUFFIX="stage-ci" + fi + ;; + tokens) + # Only julia-ci has a tokens role (see ops/terraform/iam.tf): PR + # builds and the julia-buildkite self-test run untrusted code that + # could exfiltrate any bearer token available to the job. + if [[ "${BUILDKITE_PIPELINE_SLUG:-}" != "julia-ci" || "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + echo "ERROR: bearer tokens are only available to julia-ci branch builds" >&2 + return 1 2>/dev/null || exit 1 + fi + _OIDC_ROLE_SUFFIX="tokens-ci" + ;; +esac + +# The *_id tags carry the Buildkite UUIDs (not the renameable slugs); the +# IAM trust policies pin organization_id / pipeline_id / cluster_id so a +# recreated or renamed pipeline with a matching slug cannot assume a role. +case "${_OIDC_ROLE_SUFFIX}" in + stage-pr|stage-ci|stage-buildkite) + # Trust: org/pipeline/cluster IDs. Permission policy: own commit path. + _OIDC_AWS_SESSION_TAGS="organization_id,pipeline_id,cluster_id,build_commit" + ;; + publish|docs-deploy) + # Trust: org/pipeline/cluster IDs and step key. KMS policies also gate + # the trusted roles by step_key. + _OIDC_AWS_SESSION_TAGS="organization_id,pipeline_id,cluster_id,step_key" + ;; + tokens-ci|publish-test) + # Trust only needs the immutable Buildkite IDs. + _OIDC_AWS_SESSION_TAGS="organization_id,pipeline_id,cluster_id" + ;; + *) + echo "ERROR: unknown OIDC role suffix '${_OIDC_ROLE_SUFFIX}'" >&2 + return 1 2>/dev/null || exit 1 + ;; +esac + +# Buildkite caps the OIDC token lifetime at 7200s (2h). The AWS SDK/CLI +# re-assumes the role from the token file whenever the (1h) STS session +# expires, so jobs whose AWS usage spans more than ~2h must re-source +# this script to mint a fresh token (publish.sh does so per triplet). +_OIDC_TOKEN_FILE="$(mktemp)" +# Agent v3.104+ redacts the token from logs via the Job API (a unix socket), +# which is unreachable inside the publish sandbox and aborts the request. We +# write the token to a file and never echo it, so skip redaction -- but only +# on agents new enough to know the flag (older build-cluster agents reject it +# and aren't sandbox-broken). Detect by version: `--help` can't be probed, +# since on affected agents it too needs the Job API and aborts. +_OIDC_SKIP_REDACTION=() +_BK_VER="$(buildkite-agent --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+' | head -1)" +if [ -n "${_BK_VER}" ] && { [ "${_BK_VER%%.*}" -gt 3 ] || { [ "${_BK_VER%%.*}" -eq 3 ] && [ "${_BK_VER#*.}" -ge 104 ]; }; }; then + _OIDC_SKIP_REDACTION=( --skip-redaction ) +fi +buildkite-agent oidc request-token \ + --audience "sts.amazonaws.com" \ + --lifetime 7200 \ + "${_OIDC_SKIP_REDACTION[@]}" \ + --aws-session-tag "${_OIDC_AWS_SESSION_TAGS}" \ + > "${_OIDC_TOKEN_FILE}" + +export AWS_WEB_IDENTITY_TOKEN_FILE="${_OIDC_TOKEN_FILE}" +export AWS_ROLE_ARN="arn:aws:iam::${JULIA_CI_AWS_ACCOUNT_ID}:role/julia-oidc-${_OIDC_ROLE_SUFFIX}" +export AWS_ROLE_SESSION_NAME="bk-$(tr -dc 'a-zA-Z0-9=,.@-' <<<"${BUILDKITE_STEP_KEY:-job}" | cut -c1-48)-${BUILDKITE_BUILD_NUMBER:-0}" +export AWS_DEFAULT_REGION="${JULIA_CI_AWS_REGION}" +export AWS_REGION="${JULIA_CI_AWS_REGION}" + +# Make sure stale static credentials can never shadow the role. +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + +echo "AWS credentials: ${AWS_ROLE_ARN} (via Buildkite OIDC)" diff --git a/utilities/build_envs.sh b/utilities/build_envs.sh index 1744dacd1..b70aaf38c 100755 --- a/utilities/build_envs.sh +++ b/utilities/build_envs.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# (Sourced file: deliberately no `set` of shell options here -- they would +# leak into the calling script; strict mode belongs to the entrypoints.) # First, extract information from our triplet # shellcheck source=SCRIPTDIR/extract_triplet.sh @@ -77,8 +79,9 @@ case "${ARCH?}" in "carmel,clone_all" # Apple M1 "apple-m1,base(3)" - # Vector-length-agnostic common denominator between Neoverse V1 and V2, recent Arm server architectures - "neoverse-512tvb,base(3)" + # Vector-length-agnostic common denominator between Neoverse V1 and V2, recent Arm server architectures. + # Disable little used rand and fpac features which would prevent matching on many systems. + "neoverse-512tvb,-rand,-fpac,base(3)" ) ;; esac @@ -105,8 +108,6 @@ fi JULIA_CPU_TARGET="$(printf ";%s" "${JULIA_CPU_TARGETS[@]}")" export JULIA_CPU_TARGET="${JULIA_CPU_TARGET:1}" -export JULIA_IMAGE_THREADS="$JULIA_CPU_THREADS" - # Extract git information SHORT_COMMIT_LENGTH=10 @@ -166,14 +167,20 @@ fi export JULIA_INSTALL_DIR="julia-${TAR_VERSION}" JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia${EXE}" -# By default, we upload to `julialangnightlies/bin`, but we allow this to be overridden -S3_BUCKET="${S3_BUCKET:-julialangnightlies}" -S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}" +# By default, we upload to `julialangnightlies/bin` (no-GPL builds to +# `julialang-nogpl/bin-nogpl`), but we allow this to be overridden +if [[ "${OS?}" == *nogpl ]]; then + S3_BUCKET="${S3_BUCKET:-julialang-nogpl}" + S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin-nogpl}" +else + S3_BUCKET="${S3_BUCKET:-julialangnightlies}" + S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}" +fi # We generally upload to multiple upload targets UPLOAD_TARGETS=() -if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release-* ]] || [[ "${BUILDKITE_TAG:-}" == v* ]] || [[ "${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then +if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release-* ]] || [[ "${BUILDKITE_TAG:-}" == v* ]] || [[ "${BUILDKITE_PIPELINE_SLUG}" == "julia-publish" ]]; then # First, we have the canonical fully-specified upload target UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}" ) @@ -243,23 +250,60 @@ if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release- fi fi -# If we're a pull request build, upload to a special `-prXXXX` location -if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-pr${BUILDKITE_PULL_REQUEST}-${OS?}-${ARCH?}" ) -fi - # This is the "main" filename that is used. We technically don't need this for uploading, # but it's very convenient for shuttling binaries between buildkite steps. export UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}" +# Staging target: the build step writes the unsigned tarball (write-once, +# directly from the build job) to a commit-sha-gated path in a per-pipeline +# EPHEMERAL bucket (lifecycle-expired; created by ops/terraform). julia-pr +# and julia-ci stage to SEPARATE buckets, and the trusted publish pipeline +# only ever reads the julia-ci bucket -- so a pull-request build can never +# place (or, since paths are write-once, pre-claim) anything the publish +# pipeline would consume. IAM additionally only lets a build write below +# its own commit's staging path within its own pipeline's bucket. +# - For pull requests the pr bucket is the final consumable location +# (juliaup resolves PR number -> head sha via the GitHub API and +# fetches from here). +# - For master/release/tag builds the ci bucket is the input the TRUSTED +# publish pipeline reads, signs, and promotes to the canonical +# UPLOAD_TARGETS. +# The prefix separates the normal / no-GPL artifact families within each +# bucket. +if [[ "${BUILDKITE_PIPELINE_SLUG:-}" == "julia-pr" ]]; then + STAGING_BUCKET="${STAGING_BUCKET:-julialang-ephemeral-pr}" +elif [[ "${BUILDKITE_PIPELINE_SLUG:-}" == julia-buildkite* ]]; then + # The julia-buildkite repository's own self-test CI stages to its own + # bucket, which nothing (juliaup, julia-publish) ever consumes. + STAGING_BUCKET="${STAGING_BUCKET:-julialang-ephemeral-buildkite}" +else + # julia-ci stages here; julia-publish reads its staged artifacts back + STAGING_BUCKET="${STAGING_BUCKET:-julialang-ephemeral-ci}" +fi +export STAGING_BUCKET +# Keyed by BUILDKITE_COMMIT: IAM only allows writing below the +# Buildkite-attested build_commit session tag, and BUILDKITE_COMMIT is +# that same attested value (the pipelines build JuliaLang/julia, so it is +# the julia commit). `git rev-parse HEAD` would normally agree, but the +# attested value is the one that cannot drift from the IAM condition. +# The publish trigger and deploy_docs pass BUILDKITE_COMMIT along, so +# reads agree with what was staged. +export STAGING_TARGET="${STAGING_BUCKET}/${S3_BUCKET_PREFIX}/${BUILDKITE_COMMIT?}/${UPLOAD_FILENAME}" + echo "--- Print the full and short commit hashes" echo "The full commit is: ${LONG_COMMIT}" echo "The short commit is: ${SHORT_COMMIT}" echo "Julia will be installed to: ${JULIA_BINARY}" echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})" echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})" -echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}.tar.gz" -echo "With additional upload targets:" -for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]:1}"; do - echo " -> s3://${UPLOAD_TARGET}.tar.gz" -done +# PR (and other non-master/release/tag) builds have no canonical upload targets: +# they only stage to the ephemeral bucket, so UPLOAD_TARGETS is empty here. +if [[ "${#UPLOAD_TARGETS[@]}" -gt 0 ]]; then + echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}.tar.gz" + echo "With additional upload targets:" + for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]:1}"; do + echo " -> s3://${UPLOAD_TARGET}.tar.gz" + done +else + echo "Staging only (no canonical upload targets): s3://${STAGING_TARGET}.tar.gz" +fi diff --git a/utilities/build_julia.sh b/utilities/build_julia.sh index dee1bc438..d460a76d8 100755 --- a/utilities/build_julia.sh +++ b/utilities/build_julia.sh @@ -12,6 +12,9 @@ source .buildkite/utilities/build_envs.sh # shellcheck source=SCRIPTDIR/word.sh source .buildkite/utilities/word.sh +# Build jobs emit the sysimage from one process, so use the full CPU budget. +export JULIA_IMAGE_THREADS="${JULIA_CPU_THREADS}" + echo "--- Print software versions" uname -a echo @@ -64,9 +67,23 @@ for FLAG in "${MFLAGS[@]}"; do echo " -> ${FLAG}" done +# Stream-filter the build log, rewriting the build dir to [buildroot]. On Windows +# the buildkite agent has no PTY, so this pipe is block-buffered: a long, nearly +# silent step (e.g. the ~25 min libjulia-codegen.dll link) produces no visible +# output until the buffer fills, making the build look hung. GNU sed -u flushes +# per line so the log streams live. (BSD sed on macOS has no -u, and unix jobs run +# under a PTY anyway, so only Windows needs this.) +filter_buildroot() { + if [[ "${TRIPLET}" == *mingw* ]]; then + sed -u "s|$(pwd)|[buildroot]|g" + else + sed "s|$(pwd)|[buildroot]|g" + fi +} + echo "--- Build Julia" echo "Note: The log stream is filtered. [buildroot] replaces pwd $(pwd)" -${MAKE} "${MFLAGS[@]}" 2>&1 | sed "s|$(pwd)|[buildroot]|g" +${MAKE} "${MFLAGS[@]}" 2>&1 | filter_buildroot echo "--- Check that the working directory is clean" @@ -89,7 +106,7 @@ ${JULIA_EXE} -e "import Test; Test.@test Sys.WORD_SIZE == ${EXPECTED_WORD_SIZE:? echo "--- Show build stats" echo "Note: The log stream is filtered. [buildroot] replaces pwd $(pwd)" -${MAKE} "${MFLAGS[@]}" build-stats 2>&1 | sed "s|$(pwd)|[buildroot]|g" +${MAKE} "${MFLAGS[@]}" build-stats 2>&1 | filter_buildroot echo "--- Create build artifacts" ${MAKE} "${MFLAGS[@]}" binary-dist @@ -100,4 +117,35 @@ if [[ "${JULIA_BINARYDIST_FILENAME}.tar.gz" != "${UPLOAD_FILENAME}.tar.gz" ]]; t fi echo "--- Upload build artifacts to buildkite" +# Other jobs in this build (tests, misc checks) consume the tarball as a +# buildkite artifact. buildkite-agent artifact upload "${UPLOAD_FILENAME}.tar.gz" + +echo "--- Stage unsigned tarball to s3://${STAGING_TARGET}.tar.gz" +# Stage straight from the build job (no relay through buildkite artifacts): +# a write-once upload to this pipeline's ephemeral staging bucket, gated by +# this build's commit sha. The untrusted `stage` role can do nothing else. +# shellcheck source=SCRIPTDIR/aws_oidc.sh +source .buildkite/utilities/aws_oidc.sh stage +# shellcheck source=SCRIPTDIR/upload_to_s3.sh +source .buildkite/utilities/upload_to_s3.sh +# The staging buckets disable object ACLs (public read via bucket policy) +UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.tar.gz" "${STAGING_TARGET}.tar.gz" + +# macOS: assemble the Julia.app here and stage it too. The build runs on a Mac, +# so contrib/mac/app's tooling (osacompile, etc.) is available; the trusted +# publish step then only has to codesign + repackage the .app into the signed +# .dmg -- it needs no app-building tools and no Mac. The .app is staged +# UNSIGNED (MACOS_CODESIGN_IDENTITY is unset) under a separate key; the tree +# tarball above is unchanged (test jobs still consume it). +if [[ "${OS}" == "macos" || "${OS}" == "macosnogpl" ]]; then + echo "--- [mac] Assemble the unsigned Julia.app" + # Pass the same MFLAGS as the main build (esp. TAGGED_RELEASE_BANNER): the + # contrib/mac/app rule re-runs binary-dist, and without the matching flags + # build_h.jl regenerates with a different banner, going stale and forcing a + # full system-image rebuild. With them it collapses to a fast re-install+re-tar. + MACOS_CODESIGN_IDENTITY="" ${MAKE} "${MFLAGS[@]}" -C contrib/mac/app "dmg/Julia-${MAJMIN?}.app" + tar zcf "${UPLOAD_FILENAME}.app.tar.gz" -C contrib/mac/app/dmg "Julia-${MAJMIN?}.app" + echo "--- [mac] Stage the unsigned .app to s3://${STAGING_TARGET}.app.tar.gz" + UPLOAD_TO_S3_ACL=none upload_to_s3 "${UPLOAD_FILENAME}.app.tar.gz" "${STAGING_TARGET}.app.tar.gz" +fi diff --git a/utilities/deploy_docs.sh b/utilities/deploy_docs.sh deleted file mode 100644 index a6db09e8c..000000000 --- a/utilities/deploy_docs.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Get the `.buildkite/utilities/docs` folder path -# DOCS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -echo "--- Download built docs" -buildkite-agent artifact download --step "doctest" "julia-*-htmldocs.tar.gz" - -echo "--- Deploy docs" -DOCUMENTER_KEY="$(cat .buildkite/secrets/ssh_docs_deploy)" -export DOCUMENTER_KEY -echo "Do something here!" diff --git a/utilities/extract_triplet.sh b/utilities/extract_triplet.sh index 7c849ca2a..344b6b23c 100755 --- a/utilities/extract_triplet.sh +++ b/utilities/extract_triplet.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# (Sourced file: deliberately no `set` of shell options here -- they would +# leak into the calling script; strict mode belongs to the entrypoints.) # Extract information from our triplet # Here is an example of a triplet: `x86_64-linux-gnu` diff --git a/utilities/kms_gpg_sign.py b/utilities/kms_gpg_sign.py new file mode 100755 index 000000000..6b02b277a --- /dev/null +++ b/utilities/kms_gpg_sign.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +# This file is a part of Julia. License is MIT: https://julialang.org/license +"""Create OpenPGP (GPG-compatible) detached signatures with AWS KMS. + +This replaces `gpg --detach-sig` for tarball signing. The OpenPGP packet +structure is assembled locally, but the raw RSA signature is produced by +AWS KMS (`kms:Sign` with RSASSA_PKCS1_V1_5_SHA_256). The signing key is +generated inside KMS and never leaves it; the matching OpenPGP public +key (including its self-certification, built with the certificate +helpers below) is exported once with ops/20_export_gpg_pubkey.py and +published as the Julia release signing key. + +The public half of the signing key is read from a normal GPG public key +file (armored or binary) and is only used to derive the issuer +fingerprint/key ID embedded in the signature (and to sanity check the +signature against the modulus). + +AWS credentials are resolved by the AWS CLI; in CI they come from +Buildkite OIDC web identity federation (see utilities/aws_oidc.sh). + +Usage: + kms_gpg_sign.py --public-key tarball_signing.pub.asc \ + --kms-key-id alias/julia-tarball-signing \ + FILE [FILE...] + +Produces FILE.asc for each FILE. + +For testing without KMS access, `--local-key key.pem` signs with a local +RSA private key via openssl, exercising the identical packet construction +path (openssl pkeyutl consumes the same SHA-256 digest KMS would). + +Only RSA signing keys and SHA-256 are supported. +""" + +import argparse +import base64 +import datetime +import hashlib +import os +import struct +import subprocess +import sys +import tempfile +import time + +# OpenPGP constants (RFC 4880) +PKT_SIGNATURE = 2 +PKT_PUBLIC_KEY = 6 +PKT_USER_ID = 13 +SIG_BINARY = 0x00 +SIG_POSITIVE_CERT = 0x13 +ALGO_RSA = 1 # also accept 3 (RSA sign-only); 2 (encrypt-only) is rejected +HASH_SHA256 = 8 +SUBPKT_CREATION_TIME = 2 +SUBPKT_ISSUER_KEY_ID = 16 +SUBPKT_KEY_FLAGS = 27 +SUBPKT_ISSUER_FINGERPRINT = 33 + + +def dearmor(data): + """Extract binary packet data from an ASCII-armored block, or pass through binary.""" + if not data.lstrip().startswith(b"-----BEGIN PGP"): + return data + + lines = data.decode("ascii", "replace").splitlines() + b64 = [] + in_body = False + for line in lines: + line = line.strip() + if line.startswith("-----BEGIN PGP"): + in_body = True + continue + if line.startswith("-----END PGP"): + break + if not in_body: + continue + if not line: + # blank line separates armor headers from body + b64 = [] + continue + if line.startswith("="): + # CRC24 line terminates the body + break + if ":" in line and not b64: + # armor header (Version:, Comment:, ...) + continue + b64.append(line) + + return base64.b64decode("".join(b64)) + + +def iter_packets(data): + """Yield (tag, body) for each OpenPGP packet in data.""" + i = 0 + n = len(data) + while i < n: + ctb = data[i] + if not ctb & 0x80: + raise ValueError(f"invalid packet header byte {ctb:#x} at offset {i}") + i += 1 + if ctb & 0x40: + # New format + tag = ctb & 0x3F + first = data[i] + i += 1 + if first < 192: + length = first + elif first < 224: + length = ((first - 192) << 8) + data[i] + 192 + i += 1 + elif first == 255: + length = struct.unpack(">I", data[i : i + 4])[0] + i += 4 + else: + raise ValueError("partial body lengths not supported") + else: + # Old format + tag = (ctb >> 2) & 0x0F + ltype = ctb & 0x03 + if ltype == 0: + length = data[i] + i += 1 + elif ltype == 1: + length = struct.unpack(">H", data[i : i + 2])[0] + i += 2 + elif ltype == 2: + length = struct.unpack(">I", data[i : i + 4])[0] + i += 4 + else: + raise ValueError("indeterminate length packets not supported") + yield tag, data[i : i + length] + i += length + + +def read_mpi(body, offset): + """Read an OpenPGP MPI; returns (int, new_offset).""" + bits = struct.unpack(">H", body[offset : offset + 2])[0] + nbytes = (bits + 7) // 8 + value = int.from_bytes(body[offset + 2 : offset + 2 + nbytes], "big") + return value, offset + 2 + nbytes + + +def encode_mpi(value): + """Encode an integer as an OpenPGP MPI.""" + bits = value.bit_length() + return struct.pack(">H", bits) + value.to_bytes((bits + 7) // 8, "big") + + +class PublicKeyInfo: + def __init__(self, body): + if body[0] != 4: + raise ValueError(f"only v4 keys supported (got v{body[0]})") + algo = body[5] + if algo not in (1, 3): + raise ValueError(f"only RSA signing keys supported (algorithm {algo})") + self.algorithm = algo + self.n, off = read_mpi(body, 6) + self.e, _ = read_mpi(body, off) + self.fingerprint = hashlib.sha1( + b"\x99" + struct.pack(">H", len(body)) + body + ).digest() + self.key_id = self.fingerprint[-8:] + + +def load_public_key(path): + data = dearmor(open(path, "rb").read()) + for tag, body in iter_packets(data): + if tag == PKT_PUBLIC_KEY: + return PublicKeyInfo(body) + raise ValueError(f"no public key packet found in {path}") + + +def encode_subpacket(sptype, body): + data = bytes([sptype]) + body + if len(data) < 192: + return bytes([len(data)]) + data + if len(data) < 8384: + v = len(data) - 192 + return bytes([192 + (v >> 8), v & 0xFF]) + data + return b"\xff" + struct.pack(">I", len(data)) + data + + +def encode_packet(tag, body): + """Encode a packet with a new-format header.""" + if len(body) < 192: + length = bytes([len(body)]) + elif len(body) < 8384: + v = len(body) - 192 + length = bytes([192 + (v >> 8), v & 0xFF]) + else: + length = b"\xff" + struct.pack(">I", len(body)) + return bytes([0xC0 | tag]) + length + body + + +def crc24(data): + crc = 0xB704CE + for byte in data: + crc ^= byte << 16 + for _ in range(8): + crc <<= 1 + if crc & 0x1000000: + crc ^= 0x1864CFB + return crc & 0xFFFFFF + + +def armor(packet_data, block_type="SIGNATURE"): + b64 = base64.b64encode(packet_data).decode("ascii") + lines = [b64[i : i + 64] for i in range(0, len(b64), 64)] + crc = base64.b64encode(crc24(packet_data).to_bytes(3, "big")).decode("ascii") + return ( + f"-----BEGIN PGP {block_type}-----\n\n" + + "\n".join(lines) + + f"\n={crc}\n-----END PGP {block_type}-----\n" + ) + + +class KmsSigner: + """Sign SHA-256 digests with RSASSA_PKCS1_V1_5_SHA_256 via the AWS CLI.""" + + def __init__(self, key_id): + self.key_id = key_id + + def sign_digest(self, digest): + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + f.write(digest) + digest_path = f.name + try: + out = subprocess.run( + [ + "aws", "kms", "sign", + "--key-id", self.key_id, + "--message", f"fileb://{digest_path}", + "--message-type", "DIGEST", + "--signing-algorithm", "RSASSA_PKCS1_V1_5_SHA_256", + "--output", "text", + "--query", "Signature", + ], + check=True, + capture_output=True, + ) + finally: + os.unlink(digest_path) + return base64.b64decode(out.stdout.strip()) + + +class LocalOpensslSigner: + """Sign SHA-256 digests with a local RSA key via openssl (for testing). + + Uses `openssl pkeyutl` in digest mode, which performs the same + EMSA-PKCS1-v1_5 encoding of a caller-provided digest that KMS does. + """ + + def __init__(self, key_path): + self.key_path = key_path + + def sign_digest(self, digest): + out = subprocess.run( + [ + "openssl", "pkeyutl", "-sign", + "-inkey", self.key_path, + "-pkeyopt", "digest:sha256", + "-pkeyopt", "rsa_padding_mode:pkcs1", + ], + input=digest, + check=True, + capture_output=True, + ) + return out.stdout + + +def build_signature_packet(signer, pubkey, data_hasher, sig_type=SIG_BINARY, + timestamp=None, extra_hashed_subpkts=b""): + """Build a v4 signature packet. + + data_hasher is a hashlib object already updated with the data to be + signed (document for binary sigs; key/uid data for certifications). + """ + if timestamp is None: + timestamp = int(time.time()) + + hashed_subpkts = b"".join([ + encode_subpacket(SUBPKT_ISSUER_FINGERPRINT, b"\x04" + pubkey.fingerprint), + encode_subpacket(SUBPKT_CREATION_TIME, struct.pack(">I", timestamp)), + ]) + extra_hashed_subpkts + unhashed_subpkts = encode_subpacket(SUBPKT_ISSUER_KEY_ID, pubkey.key_id) + + hashed_portion = ( + bytes([4, sig_type, ALGO_RSA, HASH_SHA256]) + + struct.pack(">H", len(hashed_subpkts)) + + hashed_subpkts + ) + trailer = b"\x04\xff" + struct.pack(">I", len(hashed_portion)) + + data_hasher.update(hashed_portion + trailer) + digest = data_hasher.digest() + + raw_signature = signer.sign_digest(digest) + + # Sanity check the raw signature against the public key before publishing: + # s^e mod n must be a valid EMSA-PKCS1-v1_5 encoding ending in our digest. + decoded = pow(int.from_bytes(raw_signature, "big"), pubkey.e, pubkey.n) + decoded_bytes = decoded.to_bytes((pubkey.n.bit_length() + 7) // 8, "big") + if not decoded_bytes.endswith(digest): + raise RuntimeError( + "signature does not verify against the supplied public key; " + "are --public-key and the KMS key the same key pair?" + ) + + body = ( + hashed_portion + + struct.pack(">H", len(unhashed_subpkts)) + + unhashed_subpkts + + digest[:2] + + encode_mpi(int.from_bytes(raw_signature, "big")) + ) + return encode_packet(PKT_SIGNATURE, body) + + +def build_rsa_key_body(n, e, timestamp): + """Build a v4 public key packet body for an RSA key. + + The fingerprint (and thus key ID) covers the creation timestamp, so + the same (n, e, timestamp) always yields the same key identity. + """ + return ( + bytes([4]) + + struct.pack(">I", timestamp) + + bytes([ALGO_RSA]) + + encode_mpi(n) + + encode_mpi(e) + ) + + +# ---- Deriving the public-key identity from the KMS key's public half -------- +# These let the signer reconstruct the OpenPGP public key (and thus the issuer +# fingerprint embedded in signatures) at runtime from `kms:GetPublicKey`, +# instead of reading a committed .asc -- useful for throwaway/test keys where +# publishing a public key block is pointless. ops/20_export_gpg_pubkey.py reuses +# these to build the full published certificate for the production key. + +def parse_der(data, offset=0): + """Parse one DER TLV; returns (tag, value, next_offset).""" + tag = data[offset] + length = data[offset + 1] + offset += 2 + if length & 0x80: + nbytes = length & 0x7F + length = int.from_bytes(data[offset : offset + nbytes], "big") + offset += nbytes + return tag, data[offset : offset + length], offset + length + + +def rsa_components_from_spki(der): + """Extract (n, e) from a SubjectPublicKeyInfo DER blob.""" + _, spki, _ = parse_der(der) # SEQUENCE SubjectPublicKeyInfo + _, _, off = parse_der(spki) # SEQUENCE AlgorithmIdentifier + tag, bits, _ = parse_der(spki, off) # BIT STRING subjectPublicKey + if tag != 0x03 or bits[0] != 0: + raise ValueError("unexpected SPKI structure (not an RSA key?)") + _, rsa, _ = parse_der(bits[1:]) # SEQUENCE RSAPublicKey + tag_n, n_bytes, off = parse_der(rsa) # INTEGER n + tag_e, e_bytes, _ = parse_der(rsa, off) # INTEGER e + if tag_n != 0x02 or tag_e != 0x02: + raise ValueError("unexpected RSAPublicKey structure") + return int.from_bytes(n_bytes, "big"), int.from_bytes(e_bytes, "big") + + +def kms_public_key(key_id): + """Fetch a KMS key's SubjectPublicKeyInfo DER via kms:GetPublicKey.""" + out = subprocess.run( + ["aws", "kms", "get-public-key", "--key-id", key_id, + "--output", "text", "--query", "PublicKey"], + check=True, capture_output=True, + ) + return base64.b64decode(out.stdout.strip()) + + +def local_public_key(key_path): + """SubjectPublicKeyInfo DER for a local RSA private key (testing only).""" + out = subprocess.run( + ["openssl", "rsa", "-in", key_path, "-pubout", "-outform", "DER"], + check=True, capture_output=True, + ) + return out.stdout + + +def kms_key_creation_date(key_id): + """The KMS key's creation time as a unix timestamp (kms:DescribeKey).""" + out = subprocess.run( + ["aws", "kms", "describe-key", "--key-id", key_id, + "--query", "KeyMetadata.CreationDate", "--output", "text"], + check=True, capture_output=True, + ).stdout.decode().strip() + # AWS CLI prints this either as epoch seconds (possibly fractional) or as an + # ISO-8601 string, depending on cli_timestamp_format; accept both. + try: + return int(float(out)) + except ValueError: + return int(datetime.datetime.fromisoformat(out.replace("Z", "+00:00")).timestamp()) + + +def parse_created(value): + """A unix timestamp, or a YYYY-MM-DD date interpreted as UTC midnight.""" + try: + return int(value) + except ValueError: + dt = datetime.datetime.strptime(value, "%Y-%m-%d") + return int(dt.replace(tzinfo=datetime.timezone.utc).timestamp()) + + +def public_key_info_from_kms(key_id, timestamp, local_key=None): + """Reconstruct the PublicKeyInfo (fingerprint, n, e) from the KMS key's + public half plus a pinned creation timestamp -- no committed file.""" + spki = local_public_key(local_key) if local_key else kms_public_key(key_id) + n, e = rsa_components_from_spki(spki) + return PublicKeyInfo(build_rsa_key_body(n, e, timestamp)) + + +def build_certificate(signer, key_body, uid, timestamp): + """Build a minimal transferable public key (certificate): public key + packet + user ID packet + positive self-certification (RFC 4880 + section 5.2.4), signed via signer. Returns (PublicKeyInfo, binary + certificate); wrap with armor(..., "PUBLIC KEY BLOCK") to publish. + """ + pubkey = PublicKeyInfo(key_body) + uid_body = uid.encode() + + hasher = hashlib.sha256() + hasher.update(b"\x99" + struct.pack(">H", len(key_body)) + key_body) + hasher.update(b"\xb4" + struct.pack(">I", len(uid_body)) + uid_body) + + # Key flags: certify + sign + key_flags = encode_subpacket(SUBPKT_KEY_FLAGS, b"\x03") + cert_sig = build_signature_packet( + signer, pubkey, hasher, sig_type=SIG_POSITIVE_CERT, + timestamp=timestamp, extra_hashed_subpkts=key_flags, + ) + + cert = ( + encode_packet(PKT_PUBLIC_KEY, key_body) + + encode_packet(PKT_USER_ID, uid_body) + + cert_sig + ) + return pubkey, cert + + +def sign_file_detached(signer, pubkey, path, output_path=None): + hasher = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + hasher.update(chunk) + + packet = build_signature_packet(signer, pubkey, hasher) + + if output_path is None: + output_path = path + ".asc" + with open(output_path, "w") as f: + f.write(armor(packet)) + return output_path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + pubgroup = parser.add_mutually_exclusive_group(required=True) + pubgroup.add_argument("--public-key", + help="GPG public key file (armored or binary) of the signing key") + pubgroup.add_argument("--public-key-from-kms", action="store_true", + help="derive the signer's OpenPGP public key identity from the " + "KMS key's public half (kms:GetPublicKey) at runtime instead " + "of a committed file; requires --created. Useful for throwaway " + "keys where publishing a public key block is pointless.") + parser.add_argument("--created", + help="override the key creation time (unix timestamp or YYYY-MM-DD " + "UTC) for --public-key-from-kms. Part of the fingerprint; " + "defaults to the KMS key's own CreationDate. Pin it only to " + "match a separately-published pubkey.") + parser.add_argument("--uid", default=None, + help="(unused for signing; accepted for symmetry with the exporter)") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--kms-key-id", + help="AWS KMS key ID/ARN/alias holding the signing key") + group.add_argument("--local-key", + help="local RSA private key PEM (testing only)") + parser.add_argument("files", nargs="+", help="files to sign (creates FILE.asc)") + args = parser.parse_args() + + if args.public_key_from_kms: + # The OpenPGP fingerprint covers the key creation timestamp. Default it + # to the KMS key's own CreationDate (so the identity reflects the real + # key); an explicit --created overrides. A local test key has no KMS + # creation date, so fall back to 0 there. + if args.created: + created = parse_created(args.created) + elif args.local_key: + created = 0 + else: + created = kms_key_creation_date(args.kms_key_id) + pubkey = public_key_info_from_kms( + args.kms_key_id, created, local_key=args.local_key) + else: + pubkey = load_public_key(args.public_key) + print(f"Signing as key {pubkey.fingerprint.hex().upper()}") + + if args.kms_key_id: + signer = KmsSigner(args.kms_key_id) + else: + signer = LocalOpensslSigner(args.local_key) + + for path in args.files: + output = sign_file_detached(signer, pubkey, path) + print(f" {path} -> {output}") + + +if __name__ == "__main__": + main() diff --git a/utilities/macos/build_dmg.sh b/utilities/macos/build_dmg.sh index 88a34e35f..742bdf1c8 100755 --- a/utilities/macos/build_dmg.sh +++ b/utilities/macos/build_dmg.sh @@ -1,78 +1,134 @@ #!/bin/bash +# Build, sign and notarize the Julia .dmg -- on LINUX -- from a PRE-ASSEMBLED, +# already-codesigned Julia.app. +# +# Separation of concerns: the build_ step (on a Mac) assembles the Julia.app +# with all of contrib/mac/app's tooling, and upload_julia.sh codesigns every +# mach-o in it (launcher + bundled julia tree) before this runs. So this step +# only wraps the signed .app in a signed, notarized .dmg -- it needs no +# app-building tooling and no Mac. +# +# Apple-only packaging is replaced as follows (the same approach Mozilla uses to +# package Firefox DMGs on linux): +# hdiutil -> newfs_hfs (the mkfs.hfsplus from hfsprogs/diskdev_cmds) to +# create the HFS+ filesystem, plus the `hfsplus` (populate) and +# `dmg` (compressed UDIF) tools from mozilla/libdmg-hfsplus. +# Code signing and notarization are linux-capable: rcodesign signs with the +# Developer ID key in AWS KMS, and notarization is an App Store Connect API +# call (key also in KMS). set -euo pipefail +THIS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# The already-codesigned Julia.app to package (see upload_julia.sh). +APP_PATH="${APP_PATH:?APP_PATH must point at the (codesigned) Julia.app}" + DMG_PATH="dmg" -mkdir -p "${DMG_PATH}" -APP_PATH="${DMG_PATH}/Julia-${MAJMIN?}.app" DMG_NAME="${UPLOAD_FILENAME?}.dmg" +VOLUME_NAME="Julia-${TAR_VERSION?}" +HFS_IMAGE="$(mktemp -u "${TMPDIR:-/tmp}/julia-dmg-XXXXXX.hfs")" + +# The Developer ID private key lives in AWS KMS; the .dmg itself is signed via +# rcodesign (see utilities/macos/rcodesign/). AWS credentials must already be +# available (source utilities/aws_oidc.sh publish). +MACOS_CODESIGN_KMS_KEY="${MACOS_CODESIGN_KMS_KEY:?}" +NOTARY_API_KEY_FILE="${THIS_DIR}/notary_api_key.json" + +# HFS+/DMG tools. newfs_hfs creates the HFS+ filesystem (the mkfs.hfsplus from +# hfsprogs/diskdev_cmds, built from source in the publish image because hfsprogs +# was dropped from Debian after bullseye); hfsplus (populate) and dmg (convert +# to UDIF) come from libdmg-hfsplus. +MKFSHFS_TOOL="${MKFSHFS_TOOL:-newfs_hfs}" +HFSPLUS_TOOL="${HFSPLUS_TOOL:-hfsplus}" +DMG_TOOL="${DMG_TOOL:-dmg}" + +# Lay out the .dmg contents: the signed .app, the /Applications symlink (added +# below; addall cannot carry it) and the volume icon (taken from inside the .app). +rm -rf "${DMG_PATH}" +mkdir -p "${DMG_PATH}" +cp -aR "${APP_PATH}" "${DMG_PATH}/" +if [[ -f "${APP_PATH}/Contents/Resources/julia.icns" ]]; then + cp "${APP_PATH}/Contents/Resources/julia.icns" "${DMG_PATH}/.VolumeIcon.icns" +fi -# Start by compiling an applescript into a `.app`, which creates the skeleton, which we will fill out -osacompile -o "${APP_PATH}" "contrib/mac/app/startup.applescript" - -# Use `plutil` to fill out the `Info.plist` appropriately -plutil -replace CFBundleDevelopmentRegion -string "en" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleDisplayName -string "Julia" "${APP_PATH}/Contents/Info.plist" -plutil -replace CFBundleIconFile -string "julia.icns" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleIdentifier -string "org.julialang.launcherapp" "${APP_PATH}/Contents/Info.plist" -plutil -replace CFBundleName -string "Julia" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleShortVersionString -string "${MAJMINPAT?}" "${APP_PATH}/Contents/Info.plist" -plutil -insert CFBundleVersion -string "${JULIA_VERSION?}-${SHORT_COMMIT?}" "${APP_PATH}/Contents/Info.plist" -plutil -insert NSHumanReadableCopyright -string "$(date '+%Y') The Julia Project" "${APP_PATH}/Contents/Info.plist" - -# Add icon file for the application and the .dmg -cp "contrib/mac/app/julia.icns" "${APP_PATH}/Contents/Resources/" -cp "contrib/mac/app/julia.icns" "${DMG_PATH}/.VolumeIcon.icns" - -# Add link to `/Applications` -ln -s /Applications "${DMG_PATH}/Applications" - -# Copy our signed tarball into the `.dmg` -cp -aR "${JULIA_INSTALL_DIR?}" "${APP_PATH}/Contents/Resources/julia" - -# Sign the `.app` launcher -.buildkite/utilities/macos/codesign.sh \ - --keychain "${KEYCHAIN_PATH}" \ - --identity "${MACOS_CODESIGN_IDENTITY}" \ - "${APP_PATH}/Contents/MacOS/applet" - -# Create `.dmg`. We create it with 1TB size, but since that is -# a maximum, it has no effect on download or unpack size. -# We define this in a function because we need to do it again later. +# Create the `.dmg`: an HFS+ filesystem image filled with the staged directory, +# converted to a compressed UDIF, then signed. function create_dmg() { - rm -f "${DMG_NAME}" - - hdiutil create \ - "${DMG_NAME}" \ - -size 1t \ - -fs HFS+ \ - -volname "Julia-${TAR_VERSION?}" \ - -imagekey zlib-level=9 \ - -srcfolder "${DMG_PATH}" + rm -f "${DMG_NAME}" "${HFS_IMAGE}" + + # Size the filesystem just above the measured tree size. The margin covers + # HFS+ catalog/extents/bitmap metadata plus per-file allocation rounding over + # the thousands of files in the Julia tree; `du -sm` already rounds each file + # up to a 4 KB block, so the real overhead is only ~5% and a 10% + 64 MB + # margin sits comfortably above the empirical failure floor (under-sizing + # fails loudly mid-populate with "rawFileWrite ... allocate", never a corrupt + # dmg). Keep it TIGHT: libdmg-hfsplus's `addall` cost scales with the image's + # total block count, not the file data -- every allocated block is zeroed and + # its bitmap byte seeked individually -- so the old 1.5x+256 MB of free space + # made the populate ~3x slower for no benefit (the final UDIF is compressed). + local size_mb + size_mb="$(( $(du -sm "${DMG_PATH}" | cut -f1) * 11 / 10 + 64 ))" + truncate -s "${size_mb}M" "${HFS_IMAGE}" + "${MKFSHFS_TOOL}" -v "${VOLUME_NAME}" "${HFS_IMAGE}" + + # `addall` logs every file and directory it copies (thousands of lines), which + # both floods the build log and slows the step (all of it streams to the + # agent). Capture it and only surface the tail on failure. + # + # `--symlinks clone_link` is REQUIRED: the Julia tree ships ~60 versioned + # dylib symlinks (libjulia.dylib -> libjulia.1.14.0.dylib, etc.). The default + # policy dereferences them into regular files, which breaks the codesign + # symlink seals (rcodesign seals them as {symlink: target}); Apple's notary + # then reports every one as "file modified" and rejects the bundle's main + # executable as having an invalid signature. clone_link recreates them as + # real HFS+ symlinks so the seals match. + local addall_log + addall_log="$(mktemp)" + if ! "${HFSPLUS_TOOL}" "${HFS_IMAGE}" addall "${DMG_PATH}" --symlinks clone_link > "${addall_log}" 2>&1; then + echo "ERROR: hfsplus addall failed; tail of its output:" >&2 + tail -30 "${addall_log}" >&2 + rm -f "${addall_log}" + return 1 + fi + rm -f "${addall_log}" + "${HFSPLUS_TOOL}" "${HFS_IMAGE}" symlink "/Applications" "/Applications" + # Mark the volume root as having a custom icon (.VolumeIcon.icns) + "${HFSPLUS_TOOL}" "${HFS_IMAGE}" attr "/" C + + "${DMG_TOOL}" build "${HFS_IMAGE}" "${DMG_NAME}" + rm -f "${HFS_IMAGE}" # Sign the `.dmg` itself - .buildkite/utilities/macos/codesign.sh \ - --keychain "${KEYCHAIN_PATH}" \ - --identity "${MACOS_CODESIGN_IDENTITY}" \ + "${THIS_DIR}/codesign.sh" \ + --kms-key "${MACOS_CODESIGN_KMS_KEY}" \ "${DMG_NAME}" } create_dmg -# Upload the `.dmg` for notarization - -xcrun notarytool \ - submit \ - --apple-id "${NOTARIZATION_APPLE_ID}" \ - --password "${NOTARIZATION_APPLE_KEY}" \ - --team-id "A427R7F42H" \ - --wait \ - "${DMG_NAME}" - -# Staple the notarization to the app -xcrun stapler staple "${APP_PATH}" +# Notarize the `.dmg`, then staple the ticket onto the `.dmg` itself. The App +# Store Connect API key also lives in KMS; notary_api_key.json contains no secret +# material (see ops/21_import_notary_key.sh). +# +# The non-production publish test stack sets PUBLISH_SKIP_NOTARIZATION=1: +# notarization is a hosted Apple round-trip with no self-signable equivalent, so +# the test pipeline skips it. The .dmg is still KMS-signed; only the Apple +# notarize + staple are skipped. +if [[ "${PUBLISH_SKIP_NOTARIZATION:-0}" != "1" ]]; then + RCODESIGN="$("${THIS_DIR}/get_rcodesign.sh")" + + "${RCODESIGN}" notary-submit \ + --api-key-file "${NOTARY_API_KEY_FILE}" \ + --wait \ + "${DMG_NAME}" -# Re-build the .dmg from the app now that it's notarized -create_dmg + # Staple the ticket onto the .dmg itself -- no rebuild needed. The stapled + # .dmg validates offline on download; Gatekeeper then assesses the .app when + # it is first launched from the mounted volume. + "${RCODESIGN}" staple "${DMG_NAME}" +else + echo "Skipping notarization (PUBLISH_SKIP_NOTARIZATION=1): .dmg is KMS-signed but not notarized/stapled." >&2 +fi # Cleanup things we created here rm -rf "${DMG_PATH}" diff --git a/utilities/macos/codesign.sh b/utilities/macos/codesign.sh index f57c9bc65..7b99eea83 100755 --- a/utilities/macos/codesign.sh +++ b/utilities/macos/codesign.sh @@ -4,107 +4,175 @@ set -euo pipefail usage() { - echo "Usage: $0 [--identity=] [--keychain=] " + echo "Usage: $0 [--kms-key=] [--certificate=] " echo echo "Parameter descriptions:" echo - echo " id: A codesigning identity hash or '-' to denote ad-hoc signing" - echo " Default value is '-'" + echo " kms-key: AWS KMS key ID/ARN/alias holding the Developer ID private key." + echo " If not given, performs ad-hoc signing with the system codesign." echo - echo " keychain_path: A path to a keychain file that contains the given identity" - echo " If not given, uses the default keychain search path." + echo " certificate: Path to the Developer ID certificate (PEM) paired with the" + echo " KMS key. Defaults to developer_id.pem next to this script." + echo + echo " For a .app directory the default is a two-phase sign: first every nested" + echo " Mach-O individually, then a bundle seal. These flags split the phases so a" + echo " caller can interpose work (e.g. patching stdlib pkgimage checksums, which" + echo " must read the *signed* pkgimages yet be sealed into CodeResources):" + echo " --no-seal: sign the nested Mach-Os but do NOT seal the bundle." + echo " --seal-only: skip the nested pass; only seal the bundle." echo echo " target: A file or directory to codesign (must come last!)" + echo + echo "When a KMS key is used, signing is performed with rcodesign (apple-codesign" + echo "with the AWS KMS backend; see utilities/macos/rcodesign/). AWS credentials" + echo "must be available, e.g. via 'source utilities/aws_oidc.sh publish'." } -abspath() { - echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")" -} +THIS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" if [ "$#" -lt 1 ]; then usage exit 1 fi -# Default codesign identity to ad-hoc signing -CODESIGN_IDENTITY="-" +KMS_KEY="" +# The Developer ID certificate (PEM) paired with the KMS key. Defaults to the +# production developer_id.pem next to this script; the non-production publish +# test stack overrides it (a self-signed cert for the test KMS key) via +# MACOS_CODESIGN_CERT. An explicit --certificate still wins over both. +CERTIFICATE="${MACOS_CODESIGN_CERT:-${THIS_DIR}/developer_id.pem}" + +# Bundle signing phases (see usage): by default a .app target gets both the +# per-file nested pass and the final bundle seal. --no-seal / --seal-only run +# only one phase so a caller can interpose work between them. +DO_NESTED=1 +DO_SEAL=1 while [ "$#" -gt 1 ]; do case "${1}" in - --identity) - CODESIGN_IDENTITY="$2" - shift - shift - ;; - --identity=*) - CODESIGN_IDENTITY="${1#*=}" - shift - ;; - --keychain) - KEYCHAIN_PATH="$2" - shift - shift - ;; - --keychain=*) - KEYCHAIN_PATH="${1#*=}" - shift - ;; + --kms-key) + KMS_KEY="$2"; shift; shift ;; + --kms-key=*) + KMS_KEY="${1#*=}"; shift ;; + --certificate) + CERTIFICATE="$2"; shift; shift ;; + --certificate=*) + CERTIFICATE="${1#*=}"; shift ;; + --no-seal) + DO_SEAL=0; shift ;; + --seal-only) + DO_NESTED=0; shift ;; *) echo "Unknown argument '$1'" usage - exit 1 - ;; + exit 1 ;; esac done +TARGET="${1}" -# Verify keychain arg -KEYCHAIN_ARGS=() -if [ -n "${KEYCHAIN_PATH:-}" ]; then - # Expand KEYCHAIN_PATH, in case it contains `~`, - # also ensuring that it is an absolute path. - KEYCHAIN_PATH="$(abspath "${KEYCHAIN_PATH}")" - if [ -f "${KEYCHAIN_PATH}" ]; then - KEYCHAIN_ARGS+=("--keychain" "${KEYCHAIN_PATH}") - - # Ensure that the given keychain has already been added to our search list and is unlocked - if ! security show-keychain-info "${KEYCHAIN_PATH}" >/dev/null 2>/dev/null; then - echo "Must unlock keychain '${KEYCHAIN_PATH}' first!" >&2 - exit 1 - fi - else - echo "Given keychain file '${KEYCHAIN_PATH}' not found!" >&2 - exit 1 - fi +if [ "${DO_NESTED}" = 0 ] && [ "${DO_SEAL}" = 0 ]; then + echo "ERROR: --no-seal and --seal-only are mutually exclusive" >&2 + exit 1 fi -# Verify that we can load the given identity -if [ "${CODESIGN_IDENTITY}" != "-" ]; then - if ! security find-identity -p codesigning "${KEYCHAIN_PATH:-}" 2>&1 | grep "${CODESIGN_IDENTITY}" >/dev/null; then - echo "security find-identity found no matching identity for '${CODESIGN_IDENTITY}':" - security find-identity -p codesigning "${KEYCHAIN_PATH:-}" - exit 1 - fi -fi +# True if $1 begins with a Mach-O magic (thin LE/BE 32/64-bit, or a fat/universal +# archive). rcodesign signs Mach-O binaries (and bundles/dmgs) only; the bundle +# tree also holds executable *non*-Mach-O files (scripts, .jl, .tbd, Makefiles +# with the +x bit) that `find -perm -0111` matches. rcodesign rejects those with +# "specified path is not of a recognized type"; they are plain data and need no +# code signature, so we skip them (see the directory loop below). +is_macho() { + local magic + magic="$(od -An -tx1 -N4 -- "${1}" 2>/dev/null | tr -d ' \n')" + case "${magic}" in + # MH_MAGIC/CIGAM (32), MH_MAGIC_64/CIGAM_64, FAT_MAGIC/CIGAM (+_64) + feedface|cefaedfe|feedfacf|cffaedfe|cafebabe|bebafeca|cafebabf|bfbafeca) return 0 ;; + *) return 1 ;; + esac +} -do_codesign() { - # Codesign with the given identity, opting into the hardened runtime, - # applying the entitlements plist we maintain within this repository, - # asking for timestamping, performing "deep" codesigning for bundles, - # and replacing any previous signatures that may exist. - codesign --sign "${CODESIGN_IDENTITY}" \ +do_adhoc_codesign() { + # Ad-hoc signing needs no key; use the system codesign as before. + codesign --sign "-" \ --option=runtime \ - --entitlements "$(dirname "${0}")/Entitlements.plist" \ - "${KEYCHAIN_ARGS[@]}" \ + --entitlements "${THIS_DIR}/Entitlements.plist" \ --timestamp \ --force \ "${1}" } -if [ -f "${1}" ]; then +do_kms_codesign() { + # Sign with the Developer ID key in AWS KMS via rcodesign. The hardened + # runtime flag and entitlements match what we passed to Apple codesign. + # rcodesign prints ~10 chatty lines per file; across hundreds of files signed + # in parallel that floods (and interleaves in) the log, so capture the output + # and only emit it if the signature actually fails. + # + # A directory target is the .app bundle: seal it WITHOUT re-signing nested + # code (--shallow). The per-file pass already signs every nested Mach-O + # individually; Apple recommends against re-signing nested code via a deep + # bundle sign, and shallow mode also avoids touching unsignable members + # (static archives, dSYM DWARF) -- they are sealed as resources. + local shallow_args=() + [[ -d "${1}" ]] && shallow_args=( --shallow ) + local out + if ! out="$("${RCODESIGN_BIN}" sign \ + --aws-kms-key "${KMS_KEY}" \ + --aws-kms-certificate-file "${CERTIFICATE}" \ + --code-signature-flags runtime \ + --entitlements-xml-file "${THIS_DIR}/Entitlements.plist" \ + ${shallow_args[@]+"${shallow_args[@]}"} \ + "${1}" 2>&1)"; then + echo "ERROR: rcodesign failed for ${1}:" >&2 + echo "${out}" >&2 + return 1 + fi +} + +if [ -n "${KMS_KEY}" ]; then + if [ ! -f "${CERTIFICATE}" ]; then + echo "ERROR: certificate '${CERTIFICATE}' not found!" >&2 + exit 1 + fi + RCODESIGN_BIN="$("${THIS_DIR}/get_rcodesign.sh")" + + # Each rcodesign process resolves AWS credentials independently. Signing a + # whole bundle means hundreds of processes, and under Buildkite OIDC each + # would run its own STS AssumeRoleWithWebIdentity. Even a small bounded pool + # overwhelmed the (proxied) STS endpoint enough that ~8% of resolutions + # exceeded the AWS SDK's 5s budget ("identity resolver timed out after 5s") + # and the signature failed. Assume the role ONCE here and hand every signer + # the resulting static session credentials via the SDK's first, network-free + # environment provider -- credential resolution becomes a local lookup that + # cannot time out, no matter how many signers run concurrently. (Only when + # running under an OIDC web-identity token; a local run with static creds + # already in the environment keeps them.) + if [[ -z "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_WEB_IDENTITY_TOKEN_FILE:-}" ]]; then + echo "Resolving shared AWS session credentials for rcodesign (one STS call for all signers)" + _kms_creds="$(aws sts assume-role-with-web-identity \ + --role-arn "${AWS_ROLE_ARN:?}" \ + --role-session-name "${AWS_ROLE_SESSION_NAME:-rcodesign}" \ + --web-identity-token "$(cat "${AWS_WEB_IDENTITY_TOKEN_FILE}")" \ + --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \ + --output text)" + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"${_kms_creds}" + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + unset _kms_creds + fi + + do_codesign() { do_kms_codesign "$@"; } + IDENTITY_DESC="KMS key ${KMS_KEY}" +else + do_codesign() { do_adhoc_codesign "$@"; } + IDENTITY_DESC="ad-hoc identity" +fi + +if [ -f "${TARGET}" ]; then # If we're codesigning a single file, directly invoke codesign on that file - echo "Codesigning file ${1} with identity ${CODESIGN_IDENTITY}" - do_codesign "${1}" -elif [ -d "${1}" ]; then + echo "Codesigning file ${TARGET} with ${IDENTITY_DESC}" + do_codesign "${TARGET}" +elif [ -d "${TARGET}" ]; then + if [ "${DO_NESTED}" = 1 ]; then # Create a fifo to communicate from `find` to `while` trap 'rm -rf $TMPFIFODIR' EXIT TMPFIFODIR="$(mktemp -d)" @@ -113,21 +181,72 @@ elif [ -d "${1}" ]; then # If we're codesigning a whole directory, use `find` to discover every # executable file within the directory, then pass that off to a while # read loop. This safely handles whitespace in filenames. - find "${1}" -type f -perm -0111 -print0 > "$TMPFIFODIR/findpipe" & + find "${TARGET}" -type f -perm -0111 -print0 > "$TMPFIFODIR/findpipe" & - # This while loop reads in from the fifo, and invokes `do_codesign`, - # but it does so in a background task, so that the codesigning can - # happen in parallel. This speeds things up by a few seconds. - echo "Codesigning dir ${1} with identity ${CODESIGN_IDENTITY}" + # Sign the discovered files with a BOUNDED pool of background workers so the + # codesigning happens in parallel without an unbounded fan-out. A full bundle + # (e.g. the Julia tree) holds hundreds of mach-o files; backgrounding a + # signer for *every* one at once spawned hundreds of simultaneous rcodesign + # processes, each independently assuming the KMS-signing OIDC role via STS + # and holding KMS / Apple-timestamp connections open for the duration of the + # signature. That much concurrent pressure pushed some credential + # resolutions past the AWS SDK's 5s budget ("identity resolver timed out + # after 5s"), failing the signature. Cap the concurrency (MACOS_CODESIGN_JOBS). + MAX_JOBS="${MACOS_CODESIGN_JOBS:-8}" + echo "Codesigning dir ${TARGET} with ${IDENTITY_DESC} (up to ${MAX_JOBS} concurrent)" + # A non-empty STATUS_DIR after the run means at least one signer failed: the + # trailing `wait` returns 0 regardless, so failures are recorded out-of-band + # rather than silently swallowed (never ship a partially-signed bundle). + STATUS_DIR="$(mktemp -d)" NUM_CODESIGNS=0 + NUM_SKIPPED=0 while IFS= read -r -d '' exe_file; do - do_codesign "${exe_file}" & + # Skip files the KMS signer (rcodesign) can't sign -- they need no + # signature, and the ad-hoc `codesign` handles them, so only filter on + # the KMS path: + # - non-Mach-O (scripts, .jl, .tbd, Makefiles, ...): rcodesign rejects + # them as "specified path is not of a recognized type". + # - static archives (*.a): a universal/"fat" archive like + # libclang_rt.osx.a carries a fat magic so it passes is_macho, but it + # is an `ar` archive, not a signable binary -- rcodesign panics + # parsing it ("index out of bounds"). Static libs are link-time input. + if [ -n "${KMS_KEY}" ] && { [[ "${exe_file}" == *.a ]] || ! is_macho "${exe_file}"; }; then + echo "Skipping non-signable file: ${exe_file}" + NUM_SKIPPED="$((NUM_SKIPPED + 1))" + continue + fi + # Block until a worker slot frees up before launching the next signer. + while (( "$(jobs -rp | wc -l)" >= MAX_JOBS )); do wait -n 2>/dev/null || true; done + { do_codesign "${exe_file}" || touch "${STATUS_DIR}/fail.${NUM_CODESIGNS}"; } & NUM_CODESIGNS="$((NUM_CODESIGNS + 1))" done < "${TMPFIFODIR}/findpipe" wait - echo "Codesigned ${NUM_CODESIGNS} files" + + NUM_FAILED="$(find "${STATUS_DIR}" -type f | wc -l)" + rm -rf "${STATUS_DIR}" + if [ "${NUM_FAILED}" -ne 0 ]; then + echo "ERROR: ${NUM_FAILED} of ${NUM_CODESIGNS} codesign operations failed" >&2 + exit 1 + fi + echo "Codesigned ${NUM_CODESIGNS} files (skipped ${NUM_SKIPPED} non-Mach-O)" + fi + + # A .app is a bundle: its main executable (Contents/MacOS/applet) must be + # signed in bundle context -- sealing Contents/_CodeSignature/CodeResources + # and binding Info.plist -- or Apple's notary rejects its signature as + # invalid. The per-file pass above signs every nested Mach-O (so the notary + # sees them all signed); this final pass seals the bundle and re-signs the + # main executable correctly. rcodesign detects the bundle from the path. + # + # When split across phases (--no-seal / --seal-only) the caller patches the + # stdlib pkgimage checksums between the nested pass and this seal, so the + # sealed CodeResources covers the final (checksum-corrected) .ji files. + if [ "${DO_SEAL}" = 1 ] && [[ "${TARGET}" == *.app && -d "${TARGET}/Contents" ]]; then + echo "Sealing .app bundle ${TARGET} with ${IDENTITY_DESC}" + do_codesign "${TARGET}" + fi else - echo "Given codesigning target '${1}' not a file or directory!" >&2 + echo "Given codesigning target '${TARGET}' not a file or directory!" >&2 usage exit 1 fi diff --git a/utilities/macos/developer_id.pem b/utilities/macos/developer_id.pem new file mode 100644 index 000000000..8a3e08d77 --- /dev/null +++ b/utilities/macos/developer_id.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF0jCCBLqgAwIBAgIQYLifKh99DyZM1MtUkk4lNjANBgkqhkiG9w0BAQsFADBe +MS0wKwYDVQQDDCREZXZlbG9wZXIgSUQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkx +CzAJBgNVBAsMAkcyMRMwEQYDVQQKDApBcHBsZSBJbmMuMQswCQYDVQQGEwJVUzAe +Fw0yNjA2MTUxODI3MzZaFw0zMTA2MTYxODI3MzVaMIGhMRowGAYKCZImiZPyLGQB +AQwKQTQyN1I3RjQySDFDMEEGA1UEAww6RGV2ZWxvcGVyIElEIEFwcGxpY2F0aW9u +OiBKdWxpYSBDb21wdXRpbmcgTExDIChBNDI3UjdGNDJIKTETMBEGA1UECwwKQTQy +N1I3RjQySDEcMBoGA1UECgwTSnVsaWEgQ29tcHV0aW5nIExMQzELMAkGA1UEBhMC +VVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKPgRbsUPdnajWEGYN +K8ip3wJuty9utiOSQwzEJTNpgQH1bZY0ITqnHYsDs+uifmUQO/eIcEvNLttJoBTX +u5xjIJr7dJqOiyeUdjdG8+Af3ahgLJ7/YgMI5BfDg54VYpvpGfQb9LdyFyhKcowx +aUBkk7Rn2g8NN1XpWFJNF2mZC35JYjUUoSDTfvuc55ELlKu/qsZ9bBK6rxIzzgVE +uSvvKkyyFxhjEvaYNq+st6nbeuJUatTehzJvbRsxAxPn1vMWnBa8iv5K3FeYyXmh +TJhGIKQzni0oZvXxvmj46ar3refzGNrrUT/4Rd8w5c7v1pg99km8t/Mvtz9PX6fl ++Q79AgMBAAGjggJGMIICQjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFPg6DGkR +duDtrNHrpln6N9XEVbAeMHIGCCsGAQUFBwEBBGYwZDAuBggrBgEFBQcwAoYiaHR0 +cDovL2NlcnRzLmFwcGxlLmNvbS9kZXZpZGcyLmRlcjAyBggrBgEFBQcwAYYmaHR0 +cDovL29jc3AuYXBwbGUuY29tL29jc3AwMy1kZXZpZGcyMDEwggEeBgNVHSAEggEV +MIIBETCCAQ0GCSqGSIb3Y2QFATCB/zCBwwYIKwYBBQUHAgIwgbYMgbNSZWxpYW5j +ZSBvbiB0aGlzIGNlcnRpZmljYXRlIGJ5IGFueSBwYXJ0eSBhc3N1bWVzIGFjY2Vw +dGFuY2Ugb2YgdGhlIHRoZW4gYXBwbGljYWJsZSBzdGFuZGFyZCB0ZXJtcyBhbmQg +Y29uZGl0aW9ucyBvZiB1c2UsIGNlcnRpZmljYXRlIHBvbGljeSBhbmQgY2VydGlm +aWNhdGlvbiBwcmFjdGljZSBzdGF0ZW1lbnRzLjA3BggrBgEFBQcCARYraHR0cHM6 +Ly93d3cuYXBwbGUuY29tL2NlcnRpZmljYXRlYXV0aG9yaXR5LzAWBgNVHSUBAf8E +DDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUHppTZm/VSGqN2etBWOQw0ReNk9owDgYD +VR0PAQH/BAQDAgeAMB8GCiqGSIb3Y2QGASEEEQwPMjAxNDA4MTEwMDAwMDBaMBMG +CiqGSIb3Y2QGAQ0BAf8EAgUAMA0GCSqGSIb3DQEBCwUAA4IBAQB1hLlmd92DFBm/ +JI1m/nfy4IQEJhZaakEV0RVRVo+JIuNWVuaD2gAukkV3dHj5L6Ju9yRyUPja7WAz +6S/e5p2LvYzosKbxNN4UxSmSNeaXMHObTCQfacv/sgiK9FN9XjcHOBKfiRiQt139 +raiwdfqRqJMaUWki+qi76xnD7ZrZTcQKk4Xz7zLXTTuWBSq62xkwqb6o+zqfwWuJ +5otNJGR1zockRUUyOa9TYw7yuFsHRA6XICBF2Y5MYULD8XboNjXoU41qONGWPXW6 +s2fFZuHR+Ix15sbNKxg5kLjxR5yPNU6asANIB1IIp3YU1gYCup592jdPjuoYDQlT +F6VAMqDL +-----END CERTIFICATE----- diff --git a/utilities/macos/developer_id_test.pem b/utilities/macos/developer_id_test.pem new file mode 100644 index 000000000..18bd6d944 --- /dev/null +++ b/utilities/macos/developer_id_test.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkDCCAnigAwIBAgIUfvASH5xsMkjkv73VJ+5sHrJvZeMwDQYJKoZIhvcNAQEL +BQAwaTE3MDUGA1UEAwwuSnVsaWEgVEVTVCBjb2Rlc2lnbmluZyBDQSAoTk9UIEZP +UiBQUk9EVUNUSU9OKTEhMB8GA1UECgwYVGhlIEp1bGlhIFByb2plY3QgKFRFU1Qp +MQswCQYDVQQGEwJVUzAeFw0yNjA2MTQxODI1MjNaFw0zNjA2MTExODI1MjNaMCEx +HzAdBgNVBAMMFkFwcGxlIENvZGUgU2lnbmluZyBDU1IwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCwCwNYOVkJQHu2MYPu9DAmc8fI3fOLRK13WQLHwgUL +unqxBroDTWbYgU0X5CJlWN9NYNCovbRPw9zPPPUIc8SZNSpeGVNKphI9H2dFLVMh +uX0W8exFFddocZs6sXrp2+CCtHVXZbn5C3LSVjz9PqMKxwV2926YyQdmzVQezhMY +hjjfOg8mKAWA5RV85xukccmUHio2RxrlnGcRfb8j01ielSSzebCNCxy7Z8ktv7g4 +DoLQZW4mpBxWV2Z2hObefpouYAD0V0S5dkO+11hhNXOX4voPRneJ1pASHPPmwAyA +6mFJB2Tn4omjC1Lc0eFTxecSLj3R+S23mGrgWCkrHc9BAgMBAAGjeDB2MA4GA1Ud +DwEB/wQEAwIHgDAWBgNVHSUBAf8EDDAKBggrBgEFBQcDAzAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTB/iMvz8TydFfOegZfq5H2H0Z9yDAfBgNVHSMEGDAWgBS4WNvN +cq8++YoBFPV7ZhDMMjoSojANBgkqhkiG9w0BAQsFAAOCAQEAP40ZEZC89YVHZks+ +rmQAdLMYW0mgL3A4mdYOCZN7NLOHgfORnS4rKSU4q0q7VRRxypny7u5raR4u9IO3 +4rW5UwUuB0SuysBp8cf6kXqCSyrqnKhthNhPGRqBFEpxDH+yCmH2Ke3zSLHC6bx+ +7f7b8f7qm1s8bBTiqwm43h5Xg+kX/NiWEduWSsLVT37mzAMtRY8wvuWF3JfMlMaD +Ol++5+7W5VrVih3SJlLpgGBTXcTy8Wpe6FlOuI4m600WAH8j7gz6ktI+GGDAhMbk +071/7zGlxOYIb+AmbBmr4GR7JyrXVWhQSL8LJpYRg981nlLULfJXq63nph3RP+/A +Jlxx5A== +-----END CERTIFICATE----- diff --git a/utilities/macos/get_rcodesign.sh b/utilities/macos/get_rcodesign.sh new file mode 100755 index 000000000..4c37e78ed --- /dev/null +++ b/utilities/macos/get_rcodesign.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Fetch (and cache) the pinned rcodesign binary; prints its path. +# +# rcodesign is apple-codesign with the AWS KMS signing backend, built from +# JuliaCI/apple-platform-rs by Yggdrasil (R/rcodesign) and published as a +# rcodesign_jll release asset (see utilities/macos/rcodesign/README.md). +# The sha256 below pins the exact tarball. It runs on the LINUX publish +# agent: rcodesign signs, notarizes and staples Apple artifacts +# cross-platform. +# +# Set RCODESIGN to override (e.g. a locally built binary). +set -euo pipefail + +RCODESIGN_VERSION="0.29.0+1" +RCODESIGN_CRATE_VERSION="0.29.0" +RCODESIGN_SHA256_x86_64_linux_gnu="9756d1cf93358e3bfcd12457450950d538c858325550cb8589f397beffbbaa06" +RCODESIGN_BASE_URL="${RCODESIGN_BASE_URL:-https://github.com/JuliaBinaryWrappers/rcodesign_jll.jl/releases/download/rcodesign-v0.29.0%2B1}" + +if [ -n "${RCODESIGN:-}" ]; then + echo "${RCODESIGN}" + exit 0 +fi + +case "$(uname -sm)" in + "Linux x86_64") TRIPLET="x86_64-linux-gnu" ;; + *) + echo "ERROR: no pinned rcodesign build for '$(uname -sm)'; set RCODESIGN" >&2 + exit 1 + ;; +esac + +SHA_VAR="RCODESIGN_SHA256_${TRIPLET//-/_}" +EXPECTED_SHA="${!SHA_VAR}" +CACHE_DIR="${HOME}/.cache/julia-buildkite" +BINARY="${CACHE_DIR}/rcodesign-${RCODESIGN_VERSION}-${TRIPLET}" + +if [ ! -x "${BINARY}" ]; then + mkdir -p "${CACHE_DIR}" + TARBALL="${BINARY}.tar.gz.tmp" + curl --fail -sSL -o "${TARBALL}" \ + "${RCODESIGN_BASE_URL}/rcodesign.v${RCODESIGN_CRATE_VERSION}.${TRIPLET}.tar.gz" >&2 + ACTUAL_SHA="$(sha256sum "${TARBALL}" | cut -d' ' -f1)" + if [ "${ACTUAL_SHA}" != "${EXPECTED_SHA}" ]; then + echo "ERROR: rcodesign sha256 mismatch (got ${ACTUAL_SHA}, expected ${EXPECTED_SHA})" >&2 + rm -f "${TARBALL}" + exit 1 + fi + EXTRACT_DIR="$(mktemp -d "${CACHE_DIR}/rcodesign-extract-XXXXXX")" + tar -xzf "${TARBALL}" -C "${EXTRACT_DIR}" bin/rcodesign + mv "${EXTRACT_DIR}/bin/rcodesign" "${BINARY}" + rm -rf "${EXTRACT_DIR}" "${TARBALL}" +fi + +echo "${BINARY}" diff --git a/utilities/macos/notary_api_key.json b/utilities/macos/notary_api_key.json new file mode 100644 index 000000000..612f5043e --- /dev/null +++ b/utilities/macos/notary_api_key.json @@ -0,0 +1,6 @@ +{ + "issuer_id": "69a6de82-7ea6-47e3-e053-5b8c7c11a4d1", + "key_id": "V8W5RG5668", + "aws_kms_key": "arn:aws:kms:us-east-1:873569884612:key/3a9c20f8-f34d-447f-b08d-043d863b7a3e", + "aws_kms_region": "us-east-1" +} diff --git a/utilities/macos/rcodesign/README.md b/utilities/macos/rcodesign/README.md new file mode 100644 index 000000000..fc49ee507 --- /dev/null +++ b/utilities/macos/rcodesign/README.md @@ -0,0 +1,48 @@ +# rcodesign with AWS KMS backend + +This directory documents the tooling for macOS codesigning + notarization +without secrets on the build agents. We use +[apple-codesign](https://github.com/indygreg/apple-platform-rs/tree/main/apple-codesign) +(`rcodesign`) with an AWS KMS signing backend, maintained on +[`JuliaCI/apple-platform-rs`](https://github.com/JuliaCI/apple-platform-rs/tree/julia-build) +(branch `julia-build`: upstream main plus the patch series listed under +Upstreaming below). + +## How it works + +* The **Developer ID Application certificate's private key** is an RSA-2048 + key held in AWS KMS (`SIGN_VERIFY`). `rcodesign sign --aws-kms-key + --aws-kms-certificate-file ` performs each low-level signature via + the KMS `Sign` API. Key material never reaches the agent. +* The **App Store Connect API key** (ECDSA P-256, used to mint ES256 JWTs + for the notary service) is also held in KMS. The unified API key JSON + (`utilities/macos/notary_api_key.json`) references the KMS key ARN and + contains **no secret material**, so it is committed in plaintext. +* AWS credentials come from Buildkite OIDC via + `AWS_WEB_IDENTITY_TOKEN_FILE`/`AWS_ROLE_ARN` (see + `utilities/aws_oidc.sh`); IAM policies restrict `kms:Sign` to the + appropriate pipeline/step session tags. + +## Building + +Yggdrasil builds the binary (recipe `R/rcodesign`, `GitSource` pinned to a +`julia-build` commit, `--features aws-kms` on all platforms) and publishes it +as [`rcodesign_jll`](https://github.com/JuliaBinaryWrappers/rcodesign_jll.jl) +release assets. CI fetches and sha256-verifies the pinned tarball via +`utilities/macos/get_rcodesign.sh`. + +To ship a new build: push the updated `julia-build` branch to +`JuliaCI/apple-platform-rs`, open a Yggdrasil PR bumping the `GitSource` +commit in `R/rcodesign/build_tarballs.jl`, and once the new `rcodesign_jll` +release exists, update the version / sha256 / base URL pins in +`get_rcodesign.sh`. + +## Upstreaming + +The patches are submitted to `indygreg/apple-platform-rs` as three series +(branches on `KenoAIStaging/apple-platform-rs`): `upstream-macho-robustness` +(zero-slice Mach-O fixes), `upstream-asc-signer` (external ES256 signers for +App Store Connect tokens), and `upstream-aws-kms` (the feature-gated +`aws-kms` backend, docs in `apple_codesign_aws_kms.rst`, unit tests). Once +they land and a release is cut, the Yggdrasil recipe returns to plain +upstream release tarballs. diff --git a/utilities/publish.sh b/utilities/publish.sh new file mode 100755 index 000000000..87de36699 --- /dev/null +++ b/utilities/publish.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Single trusted publish step. +# +# Runs once in the julia-publish pipeline and promotes every staged build +# to its final release location: it verifies the release commit, assumes the +# trusted `publish` role once, then iterates over every triplet, signing +# (macOS via rcodesign, Windows via Azure Trusted Signing, GPG tarball via +# KMS) and promoting each from the commit-sha-gated staging path to the +# canonical locations. +# +# A single LINUX step (rather than one job per platform) is feasible +# because all signing is remote-key (KMS / Trusted Signing) and every +# packaging tool is linux-capable: rcodesign signs/notarizes Apple +# artifacts cross-platform, the .dmg is built with mozilla/libdmg-hfsplus, +# the Windows installer is compiled by Inno Setup under Wine with +# Authenticode signatures from jsign, and pkgimage checksums are patched +# by a host julia. See "Publish image prerequisites" in ops/README.md. +set -euo pipefail + +# The set of arches to publish. Mirrors what the build pipeline staged. +# Each file's rows define TRIPLET (and TIMEOUT); see utilities/arches_env.sh. +ARCHES_FILES=( + .buildkite/pipelines/main/platforms/upload_linux.arches + .buildkite/pipelines/main/platforms/upload_macos.arches + .buildkite/pipelines/main/platforms/upload_windows.arches + .buildkite/pipelines/main/platforms/upload_freebsd.arches +) +# Allow callers (e.g. the scheduled no-GPL publish) to override the list. +if [[ -n "${PUBLISH_ARCHES_FILES:-}" ]]; then + # shellcheck disable=SC2206 + ARCHES_FILES=( ${PUBLISH_ARCHES_FILES} ) +fi + +# Defense in depth: refuse unless this commit is a genuine release commit on +# the canonical upstream. The real boundary is that the julia-publish +# pipeline does not build pull requests at all (see ops/README.md). +echo "--- Verify this is a trusted release commit" +bash .buildkite/utilities/verify_trusted_commit.sh + +# The trust guard above runs once; the publish role is (re-)assumed inside +# the loop below, since Buildkite OIDC tokens live at most 2h and this +# step can run longer across all triplets. +export PUBLISH_PREAUTHED=1 + +# Collect all triplets from the arches files. +TRIPLETS=() +for arches in "${ARCHES_FILES[@]}"; do + [[ -f "${arches}" ]] || { echo "WARN: missing arches file ${arches}, skipping" >&2; continue; } + while read -r env_line; do + [[ -n "${env_line}" ]] || continue + # env_line looks like: TRIPLET="x86_64-linux-gnu" TIMEOUT="30" + # shellcheck disable=SC2086 + eval "${env_line}" + [[ -n "${TRIPLET:-}" ]] && TRIPLETS+=( "${TRIPLET}" ) + done < <(bash .buildkite/utilities/arches_env.sh "${arches}") +done + +echo "--- Publishing ${#TRIPLETS[@]} triplets: ${TRIPLETS[*]}" + +# Which OIDC role to assume for signing+promotion. Defaults to the trusted +# production `publish` role; the non-production publish test stack sets +# PUBLISH_OIDC_MODE=publish-test to assume the throwaway test role instead. +PUBLISH_OIDC_MODE="${PUBLISH_OIDC_MODE:-publish}" + +FAILED=() +for triplet in "${TRIPLETS[@]}"; do + echo "+++ Publish ${triplet}" + # Fresh OIDC token per triplet (2h max lifetime; see aws_oidc.sh) + # shellcheck source=SCRIPTDIR/aws_oidc.sh + source .buildkite/utilities/aws_oidc.sh "${PUBLISH_OIDC_MODE}" + if ! TRIPLET="${triplet}" bash .buildkite/utilities/upload_julia.sh publish; then + echo "ERROR: publishing ${triplet} failed" >&2 + FAILED+=( "${triplet}" ) + fi +done + +if [[ "${#FAILED[@]}" -gt 0 ]]; then + echo "--- ${#FAILED[@]} triplet(s) failed to publish: ${FAILED[*]}" >&2 + exit 1 +fi + +echo "+++ All triplets published" diff --git a/utilities/render_launch_pipeline.py b/utilities/render_launch_pipeline.py new file mode 100644 index 000000000..2e3b572eb --- /dev/null +++ b/utilities/render_launch_pipeline.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +""" +Pre-merge renderer for the Julia Buildkite CI launch flow. + +Instead of the launch agent making ~30 separate `buildkite-agent pipeline +upload` calls (each a ~1.5s network round-trip that dominates launch latency), +this renderer produces ONE pre-grouped pipeline document on stdout. The launch +step then does a SINGLE `buildkite-agent pipeline upload` of that document. + +It reproduces, exactly, what `launch_untrusted_builders.yml` used to upload: + + * The arches-templated platform YAMLs + (pipelines/main/platforms/{build,test}_*.yml) are rendered once per arch. + The per-arch `${VAR}` / `${VAR?}` / `${VAR-default}` substitutions that the + `buildkite-agent pipeline upload` of those files used to perform are done + HERE, using the env produced by `utilities/arches_env.sh ` + plus GROUP / ALLOW_FAIL. (`USE_RR` comes from the .arches file itself where + relevant.) + + * The static / nested misc YAMLs (misc/analyzegc.yml, misc/gcext.yml, the + juliac / juliasyntax launchers, ...) are emitted VERBATIM. We do NOT + pre-resolve their variables: they contain only `$$`-runtime escapes and/or + launch-agent-env vars (e.g. `${ALLOW_FAIL?}` on gcext/test_revise). The + final single `buildkite-agent pipeline upload` of this combined document + interpolates those with the launch-agent env -- exactly as the old + per-file uploads did -- and converts `$$` -> `$`. + +CRITICAL interpolation rule: a `$$` (double dollar) is a Buildkite runtime +escape and must be PRESERVED verbatim. Per-arch substitution here only touches +single-`$` `${...}` references that are NOT preceded by another `$`. + +PowerPC: `launch_powerpc.jl` only uploads powerpc arches for Julia < 1.12. On +current master (1.14) it is a no-op, so the powerpc arches are intentionally +omitted (see OMITTED_POWERPC below). This matches the runtime behaviour. + +The result is grouped into one `group:` per label: Build, Check, Test, +Allow Fail, JuliaSyntax, JuliaC. +""" + +import os +import re +import subprocess +import sys + +# Directory of the .buildkite checkout root (this file lives in +# /utilities/render_launch_pipeline.py). +UTIL_DIR = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(UTIL_DIR) +ARCHES_ENV_SH = os.path.join(UTIL_DIR, "arches_env.sh") + +PLATFORMS = os.path.join(ROOT, "pipelines", "main", "platforms") +MISC = os.path.join(ROOT, "pipelines", "main", "misc") + + +# -------------------------------------------------------------------------- +# arches parsing (reuse arches_env.sh; do NOT reimplement .arches parsing) +# -------------------------------------------------------------------------- + +def arches_envs(arches_path): + """Return a list of dicts, one per arch row, by shelling out to + arches_env.sh and faithfully parsing each `NAME="value" ...` line via + bash `eval` so quoting matches exactly what arches_pipeline_upload.sh + would have exported.""" + out = subprocess.run( + ["bash", ARCHES_ENV_SH, arches_path], + check=True, capture_output=True, text=True, + ).stdout + + envs = [] + for line in out.splitlines(): + if not line.strip(): + continue + # The names assigned on this line (in order). + names = re.findall(r'(?:^|\s)([A-Za-z_][A-Za-z0-9_]*)=', line) + # Re-evaluate the assignment line in a clean bash so quoting is handled + # identically to `eval "export $env_map"`, then dump exactly those + # names as NUL-delimited "name=value" records to parse unambiguously + # (values may contain spaces, '=', and ','). + script = ( + 'eval "$1"; shift\n' + 'for n in "$@"; do printf "%s=%s\\0" "$n" "${!n}"; done\n' + ) + dump = subprocess.run( + ["bash", "-c", script, "_", line, *names], + check=True, capture_output=True, text=True, + ).stdout + env = {} + for rec in dump.split("\0"): + if "=" not in rec: + continue + k, v = rec.split("=", 1) + env[k] = v + envs.append(env) + return envs + + +# -------------------------------------------------------------------------- +# bash-like ${VAR} interpolation for the arches-templated YAMLs +# -------------------------------------------------------------------------- + +# Match a single-$ ${...} that is NOT preceded by another $ (i.e. not part of +# a $$ runtime escape). We assert the char before the $ is not a $. +_VAR_RE = re.compile(r'(?" +# [notify: ...] +# steps: +# - +# We want the inner step block(s) so we can re-group them. We operate on raw +# text (not a YAML round-trip) to preserve `$$` escapes and formatting exactly. +# +# These files are structurally regular, so instead of parsing them we slice the +# indented text block under a given key. The launch agents have no PyYAML, so +# the renderer must stay stdlib-only; we emit text we control (re-indented to a +# canonical depth) so indentation and `$$` stay byte-exact. + +# Canonical indentation of the emitted document: the merged inner step blocks +# live under ` steps:` of ` - group:`, so each step block's leading `- ` +# sits at 6 spaces, matching the platform sources. +STEP_INDENT = 6 + + +def _split_lines(text): + """Split into lines WITHOUT dropping a trailing newline's emptiness. + Returns the list of lines (no line endings).""" + return text.split("\n") + + +def _indent_of(line): + """Number of leading spaces; None for a blank/whitespace-only line.""" + stripped = line.lstrip(" ") + if stripped == "": + return None + return len(line) - len(stripped) + + +def _find_group_label(text, where): + """Return the group label string from the single ` - group: "