From af2b633b3868062fb3ba1643cda4054d86a91bed Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Wed, 11 Jun 2025 10:51:31 +0200 Subject: [PATCH 01/20] install diff-filter-build --- pipelines/main/launch_unsigned_jobs.yml | 241 +++++++++++-------- pipelines/scheduled/launch_unsigned_jobs.yml | 73 ++++-- 2 files changed, 183 insertions(+), 131 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index cc126493c..5579cc259 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -12,6 +12,22 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "build" + ignore: + - "*.md" + - "*.json" + - "CITATION.*" + - ".gitignore" + - ".clangd" + - ".mailmap" + # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main + # when running tests in Julia-CI/julia-buildkite, + # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master + target_branch: "master" + steps: - group: "Build" notify: @@ -33,38 +49,41 @@ steps: # that this script doesn't suddenly break when a new patch release of Julia # is released. version: '1.7.3' + - *diff-filter-build commands: | - ### Launch Linux build jobs. - # Regular: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Build" \ - ALLOW_FAIL="false" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - ### Launch macOS build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - ### Launch FreeBSD build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_freebsd.arches \ - .buildkite/pipelines/main/platforms/build_freebsd.yml - ### Launch Windows build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_windows.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + ### Launch Linux build jobs + # Regular: + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + # PowerPC (only for Julia prior to 1.12): + GROUP="Build" \ + ALLOW_FAIL="false" \ + julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ + .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + ### Launch macOS packaging jobs + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_macos.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + ### Launch FreeBSD build jobs: + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_freebsd.arches \ + .buildkite/pipelines/main/platforms/build_freebsd.yml + ### Launch Windows build jobs: + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_windows.arches \ + .buildkite/pipelines/main/platforms/build_windows.yml + fi agents: queue: "julia" os: "linux" @@ -79,17 +98,20 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + fi agents: queue: "julia" os: "linux" @@ -104,38 +126,40 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | export ALLOW_FAIL="false" - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml - ### Launch Linux test jobs. - # Regular: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # i686-linux-gnu: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.i686.arches \ - .buildkite/pipelines/main/platforms/test_linux.i686.yml - ### Launch macOS test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml - echo "+++ Finished launching test jobs" + ### Launch Linux test jobs. + # Regular: + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + # i686-linux-gnu: + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.i686.arches \ + .buildkite/pipelines/main/platforms/test_linux.i686.yml + ### Launch macOS test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_macos.arches \ + .buildkite/pipelines/main/platforms/test_macos.yml + ### Launch FreeBSD test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_freebsd.arches \ + .buildkite/pipelines/main/platforms/test_freebsd.yml + ### Launch windows test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_windows.arches \ + .buildkite/pipelines/main/platforms/test_windows.yml + fi agents: queue: "julia" os: "linux" @@ -147,18 +171,21 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail build jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS allowed-to-fail build jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + # Launch Linux allowed-to-fail build jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + # Launch macOS allowed-to-fail build jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + fi agents: queue: "julia" os: "linux" @@ -178,35 +205,37 @@ steps: # that this script doesn't suddenly break when a new patch release of Julia # is released. version: '1.7.3' + - *diff-filter-build commands: | export ALLOW_FAIL="true" - - ### Launch Linux allowed-to-fail test jobs. - # Regular: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Test" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/test_linux.powerpc.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - ### Launch macOS allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD allowed-to-fail jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + ### Launch Linux allowed-to-fail test jobs. + # Regular: + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + # PowerPC (only for Julia prior to 1.12): + GROUP="Test" \ + julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ + .buildkite/pipelines/main/platforms/test_linux.powerpc.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + ### Launch macOS allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_macos.yml + ### Launch FreeBSD allowed-to-fail jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_freebsd.yml + ### Launch windows allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_windows.yml + fi agents: queue: "julia" os: "linux" diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 413f4dc70..f4747ab8e 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -12,6 +12,22 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "build" + ignore: + - "*.md" + - "*.json" + - "CITATION.*" + - ".gitignore" + - ".clangd" + - ".mailmap" + # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main + # when running tests in Julia-CI/julia-buildkite, + # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master + target_branch: "master" + steps: - group: "Source Build" steps: @@ -20,12 +36,15 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | - GROUP="Source Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + GROUP="Source Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + fi agents: queue: "julia" os: "linux" @@ -38,11 +57,13 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - GROUP="Source Tests (Allow Fail)" \ - ALLOW_FAIL="true" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + GROUP="Source Tests (Allow Fail)" \ + ALLOW_FAIL="true" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + fi agents: queue: "julia" os: "linux" @@ -55,21 +76,23 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_windows.yml + fi agents: queue: "julia" os: "linux" From 9ebd2d7da92fad454717090cf4521c9f80177fca Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 2 Nov 2024 20:03:03 +0100 Subject: [PATCH 02/20] add diff-filter-doc --- pipelines/main/launch_unsigned_jobs.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 5579cc259..bb3f46fec 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -27,6 +27,17 @@ common: # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master target_branch: "master" + - diff-filter-doc_plugin: &diff-filter-doc + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "doc" + match: + - "doc/*" + - "HISTORY.md" + - "NEWS.md" + # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main + # when running tests in Julia-CI/julia-buildkite, + # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master + target_branch: "master" steps: - group: "Build" @@ -99,11 +110,15 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build + - *diff-filter-doc commands: | - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 || + $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_DOC} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml + fi + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml From 698c572e04b5381d40422603f65d5121e63c91c3 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Wed, 11 Jun 2025 10:52:09 +0200 Subject: [PATCH 03/20] update *upload_jobs.yml # Conflicts: # pipelines/main/launch_upload_jobs.yml --- pipelines/main/launch_upload_jobs.yml | 98 +++++++++++++--------- pipelines/scheduled/launch_upload_jobs.yml | 38 ++++++--- 2 files changed, 88 insertions(+), 48 deletions(-) diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 27f77b1cf..39f522e64 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -1,5 +1,21 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "build" + ignore: + - "*.md" + - "*.json" + - "CITATION.*" + - ".gitignore" + - ".clangd" + - ".mailmap" + # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main + # when running tests in Julia-CI/julia-buildkite, + # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master + target_branch: "master" + steps: - group: "Upload" steps: @@ -18,46 +34,49 @@ steps: # that this script doesn't suddenly break when a new patch release of Julia # is released. version: '1.7.3' + - *diff-filter-build commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="false" - ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done - ### Linux: - # Regular Linux upload jobs: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - # PowerPC Linux upload jobs, which we only run for Julia prior to 1.12: - GROUP="Upload" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/upload_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - ### macOS: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_macos.arches \ - .buildkite/pipelines/main/platforms/upload_macos.yml - ### FreeBSD: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_freebsd.arches \ - .buildkite/pipelines/main/platforms/upload_freebsd.yml - ### Windows: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_windows.arches \ - .buildkite/pipelines/main/platforms/upload_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done + ### Linux: + # Regular Linux upload jobs: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_linux.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + # PowerPC Linux upload jobs, which we only run for Julia prior to 1.12: + GROUP="Upload" \ + julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ + .buildkite/pipelines/main/platforms/upload_linux.powerpc.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + ### macOS: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_macos.arches \ + .buildkite/pipelines/main/platforms/upload_macos.yml + ### FreeBSD: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_freebsd.arches \ + .buildkite/pipelines/main/platforms/upload_freebsd.yml + ### Windows: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_windows.arches \ + .buildkite/pipelines/main/platforms/upload_windows.yml ### Upload test results.json files to buildkite - for OS in linux linux.i686 macos freebsd windows; do \ - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_$${OS}.arches \ - .buildkite/pipelines/main/misc/upload_buildkite_results.yml - done + for OS in linux macos freebsd windows; do \ + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_$${OS}.arches \ + .buildkite/pipelines/main/misc/upload_buildkite_results.yml + done + fi # Launch doctest deploy job buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml @@ -79,17 +98,20 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail upload jobs - GROUP="Allow Fail (Upload)" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + # Launch Linux allowed-to-fail upload jobs + GROUP="Allow Fail (Upload)" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + fi # Don't share this with buildkite's env display unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index c6e8a933f..17cd9d56c 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -1,5 +1,21 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "build" + ignore: + - "*.md" + - "*.json" + - "CITATION.*" + - ".gitignore" + - ".clangd" + - ".mailmap" + # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main + # when running tests in Julia-CI/julia-buildkite, + # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master + target_branch: "master" + steps: - group: "Upload (no GPL)" steps: @@ -12,16 +28,18 @@ steps: # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - # Launch `upload_*` jobs to store tarballs into S3 once tests are done - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + # Launch `upload_*` jobs to store tarballs into S3 once tests are done + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ + .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches \ + .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches \ + .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml + fi # Don't share this with buildkite's env display unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET From 70c3fd9498df442663c9d1f278907185c2230649 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 12:38:17 +0100 Subject: [PATCH 04/20] rm diff-filter-doc, but build at least x86_64-linux --- pipelines/main/launch_unsigned_jobs.yml | 88 ++++++++++------------ pipelines/main/platforms/build_freebsd.yml | 5 +- pipelines/main/platforms/build_linux.yml | 6 +- pipelines/main/platforms/build_macos.yml | 5 +- pipelines/main/platforms/build_windows.yml | 5 +- 5 files changed, 55 insertions(+), 54 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index bb3f46fec..2ce1dd5df 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -27,17 +27,6 @@ common: # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master target_branch: "master" - - diff-filter-doc_plugin: &diff-filter-doc - https://github.com/fatteneder/diff-filter-buildkite-plugin#main: - name: "doc" - match: - - "doc/*" - - "HISTORY.md" - - "NEWS.md" - # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main - # when running tests in Julia-CI/julia-buildkite, - # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master - target_branch: "master" steps: - group: "Build" @@ -62,39 +51,42 @@ steps: version: '1.7.3' - *diff-filter-build commands: | - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then - ### Launch Linux build jobs - # Regular: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Build" \ - ALLOW_FAIL="false" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - ### Launch macOS packaging jobs - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - ### Launch FreeBSD build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_freebsd.arches \ - .buildkite/pipelines/main/platforms/build_freebsd.yml - ### Launch Windows build jobs: - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_windows.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml - fi + ### Launch Linux build jobs + # Regular: + GROUP="Build" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + # PowerPC (only for Julia prior to 1.12): + GROUP="Build" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ + .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + ### Launch macOS packaging jobs + GROUP="Build" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_macos.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + ### Launch FreeBSD build jobs: + GROUP="Build" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_freebsd.arches \ + .buildkite/pipelines/main/platforms/build_freebsd.yml + ### Launch Windows build jobs: + GROUP="Build" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_windows.arches \ + .buildkite/pipelines/main/platforms/build_windows.yml agents: queue: "julia" os: "linux" @@ -110,13 +102,9 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build - - *diff-filter-doc commands: | - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 || - $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_DOC} == 1 ]]; then - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - fi + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml diff --git a/pipelines/main/platforms/build_freebsd.yml b/pipelines/main/platforms/build_freebsd.yml index df1c7a62a..849404a91 100644 --- a/pipelines/main/platforms/build_freebsd.yml +++ b/pipelines/main/platforms/build_freebsd.yml @@ -9,7 +9,10 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/build_julia.sh" + commands: | + if [[ ${BUILD?} == 1 ]]; then + bash .buildkite/utilities/build_julia.sh + fi agents: queue: "julia" os: "freebsd" diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml index f869da0cf..87f4ffc2e 100644 --- a/pipelines/main/platforms/build_linux.yml +++ b/pipelines/main/platforms/build_linux.yml @@ -21,7 +21,11 @@ steps: - "/cache/repos:/cache/repos" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/build_julia.sh" + commands: | + # always build on x86_64-linux-gnu so that we can at least build docs + if [[ ${BUILD?} == 1 || ${TRIPLET?} == "x86_64-linux-gnu" ]]; then + bash .buildkite/utilities/build_julia.sh + fi agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml index 4c4037153..eaa3cd056 100644 --- a/pipelines/main/platforms/build_macos.yml +++ b/pipelines/main/platforms/build_macos.yml @@ -9,7 +9,10 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/build_julia.sh" + commands: | + if [[ ${BUILD?} == 1 ]]; then + bash .buildkite/utilities/build_julia.sh + fi agents: queue: "julia" os: "macos" diff --git a/pipelines/main/platforms/build_windows.yml b/pipelines/main/platforms/build_windows.yml index 63839b23c..ba742b07a 100644 --- a/pipelines/main/platforms/build_windows.yml +++ b/pipelines/main/platforms/build_windows.yml @@ -10,7 +10,10 @@ steps: - docker#v3.13.0: image: "juliapackaging/package-windows-${DOCKER_ARCH?}:${DOCKER_TAG?}" always-pull: true - command: ["bash", ".buildkite/utilities/build_julia.sh"] + commands: | + if [[ ${BUILD?} == 1 ]]; then + bash .buildkite/utilities/build_julia.sh + fi propagate-environment: true volumes: # Mount buildkite-agent as well From 8d4b31ed31bdf18da72df366e2039eaca37e52ea Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 12:40:11 +0100 Subject: [PATCH 05/20] add option to force build using a PR label --- pipelines/main/launch_unsigned_jobs.yml | 34 +++++++++++++++----- pipelines/main/launch_upload_jobs.yml | 14 ++++++-- pipelines/scheduled/launch_unsigned_jobs.yml | 18 +++++++++-- pipelines/scheduled/launch_upload_jobs.yml | 10 ++++-- 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 2ce1dd5df..3df85848d 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -27,6 +27,9 @@ common: # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master target_branch: "master" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS steps: - group: "Build" @@ -50,12 +53,15 @@ steps: # is released. version: '1.7.3' - *diff-filter-build + - *pr-labels commands: | + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) ### Launch Linux build jobs # Regular: GROUP="Build" \ ALLOW_FAIL="false" \ - BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_linux.arches \ .buildkite/pipelines/main/platforms/build_linux.yml @@ -69,21 +75,21 @@ steps: ### Launch macOS packaging jobs GROUP="Build" \ ALLOW_FAIL="false" \ - BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_macos.arches \ .buildkite/pipelines/main/platforms/build_macos.yml ### Launch FreeBSD build jobs: GROUP="Build" \ ALLOW_FAIL="false" \ - BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_freebsd.arches \ .buildkite/pipelines/main/platforms/build_freebsd.yml ### Launch Windows build jobs: GROUP="Build" \ ALLOW_FAIL="false" \ - BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_windows.arches \ .buildkite/pipelines/main/platforms/build_windows.yml @@ -102,10 +108,13 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build + - *pr-labels commands: | buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml @@ -130,9 +139,12 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build + - *pr-labels commands: | export ALLOW_FAIL="false" - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml @@ -175,9 +187,12 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build + - *pr-labels commands: | export ALLOW_FAIL="true" - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then # Launch Linux allowed-to-fail build jobs GROUP="Allow Fail" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ @@ -209,9 +224,12 @@ steps: # is released. version: '1.7.3' - *diff-filter-build + - *pr-labels commands: | export ALLOW_FAIL="true" - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then ### Launch Linux allowed-to-fail test jobs. # Regular: GROUP="Allow Fail" \ diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 39f522e64..d4f24645e 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -15,6 +15,9 @@ common: # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master target_branch: "master" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS steps: - group: "Upload" @@ -35,12 +38,14 @@ steps: # is released. version: '1.7.3' - *diff-filter-build + - *pr-labels commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="false" - - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done ### Linux: # Regular Linux upload jobs: @@ -99,13 +104,16 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build + - *pr-labels commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) export ALLOW_FAIL="true" - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + if [[ $${BUILD} == 1 ]]; then # Launch Linux allowed-to-fail upload jobs GROUP="Allow Fail (Upload)" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index f4747ab8e..18f912934 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -27,6 +27,9 @@ common: # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master target_branch: "master" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS steps: - group: "Source Build" @@ -37,8 +40,11 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - *diff-filter-build + - *pr-labels commands: | - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then GROUP="Source Build" \ ALLOW_FAIL="false" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ @@ -57,7 +63,9 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE})) + if [[ ${BUILD} == 1 ]]; then GROUP="Source Tests (Allow Fail)" \ ALLOW_FAIL="true" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ @@ -75,8 +83,12 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then GROUP="no_GPL" \ ALLOW_FAIL="false" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index 17cd9d56c..566407831 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -15,6 +15,9 @@ common: # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master target_branch: "master" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS steps: - group: "Upload (no GPL)" @@ -24,11 +27,14 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - - if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + if [[ $${BUILD} == 1 ]]; then # Launch `upload_*` jobs to store tarballs into S3 once tests are done bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ From 3fbc5a266c5d257cabe12d151b1d0cc39563f93c Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 12:41:02 +0100 Subject: [PATCH 06/20] rename trigger variable --- pipelines/main/launch_unsigned_jobs.yml | 12 ++++++------ pipelines/main/launch_upload_jobs.yml | 6 +++--- pipelines/scheduled/launch_unsigned_jobs.yml | 8 ++++---- pipelines/scheduled/launch_upload_jobs.yml | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 3df85848d..09ad50cf7 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -15,7 +15,7 @@ common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: - name: "build" + name: "TRIGGER_BUILD" ignore: - "*.md" - "*.json" @@ -56,7 +56,7 @@ steps: - *pr-labels commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) ### Launch Linux build jobs # Regular: GROUP="Build" \ @@ -113,7 +113,7 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml @@ -143,7 +143,7 @@ steps: commands: | export ALLOW_FAIL="false" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml @@ -191,7 +191,7 @@ steps: commands: | export ALLOW_FAIL="true" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then # Launch Linux allowed-to-fail build jobs GROUP="Allow Fail" \ @@ -228,7 +228,7 @@ steps: commands: | export ALLOW_FAIL="true" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then ### Launch Linux allowed-to-fail test jobs. # Regular: diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index d4f24645e..76fd74e7b 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -3,7 +3,7 @@ common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: - name: "build" + name: "TRIGGER_BUILD" ignore: - "*.md" - "*.json" @@ -44,7 +44,7 @@ steps: export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="false" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done ### Linux: @@ -109,7 +109,7 @@ steps: # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) export ALLOW_FAIL="true" diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 18f912934..46d9bab94 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -15,7 +15,7 @@ common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: - name: "build" + name: "TRIGGER_BUILD" ignore: - "*.md" - "*.json" @@ -43,7 +43,7 @@ steps: - *pr-labels commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then GROUP="Source Build" \ ALLOW_FAIL="false" \ @@ -64,7 +64,7 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ ${BUILD} == 1 ]]; then GROUP="Source Tests (Allow Fail)" \ ALLOW_FAIL="true" \ @@ -87,7 +87,7 @@ steps: - *pr-labels commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then GROUP="no_GPL" \ ALLOW_FAIL="false" \ diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index 566407831..fcc25ce78 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -3,7 +3,7 @@ common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: - name: "build" + name: "TRIGGER_BUILD" ignore: - "*.md" - "*.json" @@ -33,7 +33,7 @@ steps: # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) - BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE})) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then # Launch `upload_*` jobs to store tarballs into S3 once tests are done bash .buildkite/utilities/arches_pipeline_upload.sh \ From 6ff89185b665e5c917c83e52a0c0505c8e300ba5 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Wed, 6 Nov 2024 15:01:19 +0100 Subject: [PATCH 07/20] update ignore list --- pipelines/main/launch_unsigned_jobs.yml | 7 +++---- pipelines/main/launch_upload_jobs.yml | 7 +++---- pipelines/scheduled/launch_unsigned_jobs.yml | 7 +++---- pipelines/scheduled/launch_upload_jobs.yml | 7 +++---- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 09ad50cf7..095d584c1 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -18,11 +18,10 @@ common: name: "TRIGGER_BUILD" ignore: - "*.md" - - "*.json" + - ".*" + - "julia.spdx.json" - "CITATION.*" - - ".gitignore" - - ".clangd" - - ".mailmap" + - "typos.toml" # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 76fd74e7b..e1f51a1cb 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -6,11 +6,10 @@ common: name: "TRIGGER_BUILD" ignore: - "*.md" - - "*.json" + - ".*" + - "julia.spdx.json" - "CITATION.*" - - ".gitignore" - - ".clangd" - - ".mailmap" + - "typos.toml" # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 46d9bab94..df9734108 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -18,11 +18,10 @@ common: name: "TRIGGER_BUILD" ignore: - "*.md" - - "*.json" + - ".*" + - "julia.spdx.json" - "CITATION.*" - - ".gitignore" - - ".clangd" - - ".mailmap" + - "typos.toml" # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index fcc25ce78..2dcfa27a8 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -6,11 +6,10 @@ common: name: "TRIGGER_BUILD" ignore: - "*.md" - - "*.json" + - ".*" + - "julia.spdx.json" - "CITATION.*" - - ".gitignore" - - ".clangd" - - ".mailmap" + - "typos.toml" # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main # when running tests in Julia-CI/julia-buildkite, # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master From 5ef9ed897e1c5eec25b644cb157686b0c57b9b8e Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Wed, 6 Nov 2024 16:38:31 +0100 Subject: [PATCH 08/20] add missing BUILD exports --- pipelines/main/launch_unsigned_jobs.yml | 24 +++++----- pipelines/scheduled/launch_unsigned_jobs.yml | 48 ++++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 095d584c1..b6640ec26 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -191,18 +191,18 @@ steps: export ALLOW_FAIL="true" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) - if [[ $${BUILD} == 1 ]]; then - # Launch Linux allowed-to-fail build jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS allowed-to-fail build jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - fi + # Launch Linux allowed-to-fail build jobs + GROUP="Allow Fail" \ + BUILD="$${BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + # Launch macOS allowed-to-fail build jobs + GROUP="Allow Fail" \ + BUILD="$${BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml agents: queue: "julia" os: "linux" diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index df9734108..0181a4acf 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -43,13 +43,12 @@ steps: commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) - if [[ $${BUILD} == 1 ]]; then - GROUP="Source Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - fi + GROUP="Source Build" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml agents: queue: "julia" os: "linux" @@ -87,23 +86,24 @@ steps: commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) - if [[ $${BUILD} == 1 ]]; then - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml - fi + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_windows.yml agents: queue: "julia" os: "linux" From 52f2af1aeba226ca8276e4b98e642187fe13497e Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Wed, 6 Nov 2024 19:07:56 +0100 Subject: [PATCH 09/20] provide bash script as string for docker plugin command --- pipelines/main/platforms/build_windows.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pipelines/main/platforms/build_windows.yml b/pipelines/main/platforms/build_windows.yml index ba742b07a..ca55bd58e 100644 --- a/pipelines/main/platforms/build_windows.yml +++ b/pipelines/main/platforms/build_windows.yml @@ -10,10 +10,7 @@ steps: - docker#v3.13.0: image: "juliapackaging/package-windows-${DOCKER_ARCH?}:${DOCKER_TAG?}" always-pull: true - commands: | - if [[ ${BUILD?} == 1 ]]; then - bash .buildkite/utilities/build_julia.sh - fi + command: "[[ ${BUILD?} == 1 ]] && bash .buildkite/utilities/build_julia.sh" propagate-environment: true volumes: # Mount buildkite-agent as well From 2ab621e300f13e6df823db38cc016b3ee88fd49c Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Wed, 6 Nov 2024 19:37:48 +0100 Subject: [PATCH 10/20] fix an if condition --- pipelines/scheduled/launch_unsigned_jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 0181a4acf..92f5eeb9e 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -63,7 +63,7 @@ steps: commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) - if [[ ${BUILD} == 1 ]]; then + if [[ $${BUILD} == 1 ]]; then GROUP="Source Tests (Allow Fail)" \ ALLOW_FAIL="true" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ From b8224c2803d17e47b69d3ae46902c5244893c6e3 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Thu, 7 Nov 2024 14:56:48 +0100 Subject: [PATCH 11/20] can we use the step's command instead of docker's? --- pipelines/main/platforms/build_windows.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/main/platforms/build_windows.yml b/pipelines/main/platforms/build_windows.yml index ca55bd58e..9c61d6dc5 100644 --- a/pipelines/main/platforms/build_windows.yml +++ b/pipelines/main/platforms/build_windows.yml @@ -10,7 +10,6 @@ steps: - docker#v3.13.0: image: "juliapackaging/package-windows-${DOCKER_ARCH?}:${DOCKER_TAG?}" always-pull: true - command: "[[ ${BUILD?} == 1 ]] && bash .buildkite/utilities/build_julia.sh" propagate-environment: true volumes: # Mount buildkite-agent as well @@ -22,6 +21,8 @@ steps: - "JULIA_CPU_THREADS" # Have to include this for `buildkite-agent` to work: - "BUILDKITE_AGENT_ACCESS_TOKEN" + commands: | + bash .buildkite/utilities/build_julia.sh timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} agents: From 65dff2f1851e47774213282ea92043e83d86f2ee Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 9 Nov 2024 11:59:26 +0100 Subject: [PATCH 12/20] try without multiline strings --- pipelines/main/platforms/build_windows.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/main/platforms/build_windows.yml b/pipelines/main/platforms/build_windows.yml index 9c61d6dc5..97f47ea89 100644 --- a/pipelines/main/platforms/build_windows.yml +++ b/pipelines/main/platforms/build_windows.yml @@ -21,8 +21,7 @@ steps: - "JULIA_CPU_THREADS" # Have to include this for `buildkite-agent` to work: - "BUILDKITE_AGENT_ACCESS_TOKEN" - commands: | - bash .buildkite/utilities/build_julia.sh + command: "[[ ${BUILD?} == 1 ]] && bash .buildkite/utilities/build_julia.sh;" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} agents: From 13c10cb0403c9bf62be3d42ab481489c7675ebbc Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 9 Nov 2024 18:06:39 +0100 Subject: [PATCH 13/20] try with cmd.exe if --- pipelines/main/platforms/build_windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/main/platforms/build_windows.yml b/pipelines/main/platforms/build_windows.yml index 97f47ea89..f66d2099a 100644 --- a/pipelines/main/platforms/build_windows.yml +++ b/pipelines/main/platforms/build_windows.yml @@ -21,7 +21,7 @@ steps: - "JULIA_CPU_THREADS" # Have to include this for `buildkite-agent` to work: - "BUILDKITE_AGENT_ACCESS_TOKEN" - command: "[[ ${BUILD?} == 1 ]] && bash .buildkite/utilities/build_julia.sh;" + command: "IF ${BUILD?} == 1 bash .buildkite/utilities/build_julia.sh" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} agents: From 1322934d4db463088d0b118e6b42e60b3c9d408c Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 9 Nov 2024 18:34:37 +0100 Subject: [PATCH 14/20] set diff-filter's target_branch option conditionally using environment hook --- .buildkite/hooks/post-checkout | 7 +++++++ pipelines/main/launch_unsigned_jobs.yml | 6 ++---- pipelines/main/launch_upload_jobs.yml | 6 ++---- pipelines/main/platforms/build_linux.yml | 2 +- pipelines/scheduled/launch_unsigned_jobs.yml | 6 ++---- pipelines/scheduled/launch_upload_jobs.yml | 6 ++---- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 8cdb3d7ad..d75007124 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -72,3 +72,10 @@ fi # Export S3_BUCKET and S3_BUCKET_PREFIX, to force our CI to upload to a different S3 target than the actual CI pipeline would. export S3_BUCKET="julialang-ephemeral" export S3_BUCKET_PREFIX="julia-buildkite-uploads/bin" + +# for PRs from a julia-buildkite repo we have to set the target branch manually +if [[ ${BUILDKITE_PULL_REQUEST_REPO} =~ julia-buildkite.git$ ]]; then + export DIFF_FILTER_TARGET_BRANCH="master" +else + export DIFF_FILTER_TARGET_BRANCH="" +fi diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index b6640ec26..999637397 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -22,10 +22,8 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main - # when running tests in Julia-CI/julia-buildkite, - # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master - target_branch: "master" + # exported from .buildkite/hooks/environment + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: publish-env-var: PULL_REQUEST_LABELS diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index e1f51a1cb..21494181a 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -10,10 +10,8 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main - # when running tests in Julia-CI/julia-buildkite, - # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master - target_branch: "master" + # exported from .buildkite/hooks/environment + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: publish-env-var: PULL_REQUEST_LABELS diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml index 87f4ffc2e..5c10c7955 100644 --- a/pipelines/main/platforms/build_linux.yml +++ b/pipelines/main/platforms/build_linux.yml @@ -23,7 +23,7 @@ steps: soft_fail: ${ALLOW_FAIL?} commands: | # always build on x86_64-linux-gnu so that we can at least build docs - if [[ ${BUILD?} == 1 || ${TRIPLET?} == "x86_64-linux-gnu" ]]; then + if [[ ${BUILD?} == 1 || "${TRIPLET?}" == "x86_64-linux-gnu" ]]; then bash .buildkite/utilities/build_julia.sh fi agents: diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 92f5eeb9e..51d420ae8 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -22,10 +22,8 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main - # when running tests in Julia-CI/julia-buildkite, - # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master - target_branch: "master" + # exported from .buildkite/hooks/post-checkout + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: publish-env-var: PULL_REQUEST_LABELS diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index 2dcfa27a8..023bacd11 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -10,10 +10,8 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main - # when running tests in Julia-CI/julia-buildkite, - # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master - target_branch: "master" + # exported from .buildkite/hooks/post-checkout + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: publish-env-var: PULL_REQUEST_LABELS From c44f2b71b6eca7c4bd8b52964e39cbc347c2ab92 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 10 Nov 2024 17:09:22 +0100 Subject: [PATCH 15/20] use grep -E everywhere for compatability reasons --- .buildkite/hooks/post-checkout | 2 +- pipelines/main/launch_unsigned_jobs.yml | 10 +++++----- pipelines/main/launch_upload_jobs.yml | 4 ++-- pipelines/scheduled/launch_unsigned_jobs.yml | 6 +++--- pipelines/scheduled/launch_upload_jobs.yml | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index d75007124..f87832ad6 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -74,7 +74,7 @@ export S3_BUCKET="julialang-ephemeral" export S3_BUCKET_PREFIX="julia-buildkite-uploads/bin" # for PRs from a julia-buildkite repo we have to set the target branch manually -if [[ ${BUILDKITE_PULL_REQUEST_REPO} =~ julia-buildkite.git$ ]]; then +if [[ -n "$(echo ${BUILDKITE_PULL_REQUEST_REPO} | grep -E "julia-buildkite\.git$")" ]]; then export DIFF_FILTER_TARGET_BRANCH="master" else export DIFF_FILTER_TARGET_BRANCH="" diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 999637397..a170dd458 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -52,7 +52,7 @@ steps: - *diff-filter-build - *pr-labels commands: | - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) ### Launch Linux build jobs # Regular: @@ -109,7 +109,7 @@ steps: commands: | buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml @@ -139,7 +139,7 @@ steps: - *pr-labels commands: | export ALLOW_FAIL="false" - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml @@ -187,7 +187,7 @@ steps: - *pr-labels commands: | export ALLOW_FAIL="true" - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) # Launch Linux allowed-to-fail build jobs GROUP="Allow Fail" \ @@ -224,7 +224,7 @@ steps: - *pr-labels commands: | export ALLOW_FAIL="true" - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then ### Launch Linux allowed-to-fail test jobs. diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 21494181a..5b0000542 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -40,7 +40,7 @@ steps: # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="false" - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done @@ -105,7 +105,7 @@ steps: commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) export ALLOW_FAIL="true" diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 51d420ae8..2b77f40e3 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -39,7 +39,7 @@ steps: - *diff-filter-build - *pr-labels commands: | - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) GROUP="Source Build" \ ALLOW_FAIL="false" \ @@ -59,7 +59,7 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then GROUP="Source Tests (Allow Fail)" \ @@ -82,7 +82,7 @@ steps: - *diff-filter-build - *pr-labels commands: | - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) GROUP="no_GPL" \ ALLOW_FAIL="false" \ diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index 023bacd11..84fb4aadb 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -29,7 +29,7 @@ steps: commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\")" && echo 1 || echo 0) + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) if [[ $${BUILD} == 1 ]]; then # Launch `upload_*` jobs to store tarballs into S3 once tests are done From c9a95fbf6ab78f4fd7c7bf568753b7037d676fe6 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 11 Nov 2024 16:53:59 +0100 Subject: [PATCH 16/20] add an env var for the target branch --- .buildkite/hooks/post-checkout | 2 -- pipelines/main/launch_unsigned_jobs.yml | 4 +++- pipelines/main/launch_upload_jobs.yml | 4 +++- pipelines/scheduled/launch_unsigned_jobs.yml | 4 +++- pipelines/scheduled/launch_upload_jobs.yml | 4 +++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index f87832ad6..16e6922aa 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -76,6 +76,4 @@ export S3_BUCKET_PREFIX="julia-buildkite-uploads/bin" # for PRs from a julia-buildkite repo we have to set the target branch manually if [[ -n "$(echo ${BUILDKITE_PULL_REQUEST_REPO} | grep -E "julia-buildkite\.git$")" ]]; then export DIFF_FILTER_TARGET_BRANCH="master" -else - export DIFF_FILTER_TARGET_BRANCH="" fi diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index a170dd458..f3d037256 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -12,6 +12,9 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +env: + DIFF_FILTER_TARGET_BRANCH: "" + common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: @@ -22,7 +25,6 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # exported from .buildkite/hooks/environment target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 5b0000542..bc98d4378 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -1,5 +1,8 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +env: + DIFF_FILTER_TARGET_BRANCH: "" + common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: @@ -10,7 +13,6 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # exported from .buildkite/hooks/environment target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 2b77f40e3..1c9660aa7 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -12,6 +12,9 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +env: + DIFF_FILTER_TARGET_BRANCH: "" + common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: @@ -22,7 +25,6 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # exported from .buildkite/hooks/post-checkout target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index 84fb4aadb..2cf06bed9 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -1,5 +1,8 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +env: + DIFF_FILTER_TARGET_BRANCH: "" + common: - diff-filter-build_plugin: &diff-filter-build https://github.com/fatteneder/diff-filter-buildkite-plugin#main: @@ -10,7 +13,6 @@ common: - "julia.spdx.json" - "CITATION.*" - "typos.toml" - # exported from .buildkite/hooks/post-checkout target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" - pr-labels_plugin: &pr-labels sv-oss/github-pr-labels#v0.0.2: From dc5ce912c2676d46ef73aba5061ad088e10a3b2a Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 13:26:20 +0100 Subject: [PATCH 17/20] forward env vars to upload script --- pipelines/main/platforms/launch_powerpc.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/main/platforms/launch_powerpc.jl b/pipelines/main/platforms/launch_powerpc.jl index 3d6a58ccb..29977cafd 100644 --- a/pipelines/main/platforms/launch_powerpc.jl +++ b/pipelines/main/platforms/launch_powerpc.jl @@ -15,7 +15,7 @@ const arches_file = ARGS[1] # .buildkite/pipelines/main/platforms/build_linux.yml const yaml_pipeline_file = ARGS[2] -const cmd = `bash ".buildkite/utilities/arches_pipeline_upload.sh" "$(arches_file)" "$(yaml_pipeline_file)"` +const cmd = Cmd(`bash ".buildkite/utilities/arches_pipeline_upload.sh" "$(arches_file)" "$(yaml_pipeline_file)"`, env=copy(ENV)) const parsed_julia_version = parse_version_file() const should_include_powerpc_builds = parsed_julia_version < v"1.12-" From 368cc292af32877388ca0615fe19d6548646f585 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 13:32:24 +0100 Subject: [PATCH 18/20] fix varname --- pipelines/main/launch_unsigned_jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index f3d037256..ebba9b96a 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -67,7 +67,7 @@ steps: # PowerPC (only for Julia prior to 1.12): GROUP="Build" \ ALLOW_FAIL="false" \ - BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \ + BUILD="$${BUILD}" \ julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ .buildkite/pipelines/main/platforms/build_linux.yml From e904a527392cee901c7a19c900c20e1967e4fae6 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 13:35:14 +0100 Subject: [PATCH 19/20] also trigger build if PR gets merged --- pipelines/main/launch_unsigned_jobs.yml | 15 ++++++++++----- pipelines/main/launch_upload_jobs.yml | 6 ++++-- pipelines/scheduled/launch_unsigned_jobs.yml | 9 ++++++--- pipelines/scheduled/launch_upload_jobs.yml | 3 ++- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index ebba9b96a..347554513 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -55,7 +55,8 @@ steps: - *pr-labels commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) ### Launch Linux build jobs # Regular: GROUP="Build" \ @@ -112,7 +113,8 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml @@ -142,7 +144,8 @@ steps: commands: | export ALLOW_FAIL="false" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) if [[ $${BUILD} == 1 ]]; then buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml @@ -190,7 +193,8 @@ steps: commands: | export ALLOW_FAIL="true" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) # Launch Linux allowed-to-fail build jobs GROUP="Allow Fail" \ BUILD="$${BUILD}" \ @@ -227,7 +231,8 @@ steps: commands: | export ALLOW_FAIL="true" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) if [[ $${BUILD} == 1 ]]; then ### Launch Linux allowed-to-fail test jobs. # Regular: diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index bc98d4378..07caa4c55 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -43,7 +43,8 @@ steps: export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="false" FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) if [[ $${BUILD} == 1 ]]; then ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done ### Linux: @@ -108,7 +109,8 @@ steps: # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) export ALLOW_FAIL="true" diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 1c9660aa7..db95f020c 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -42,7 +42,8 @@ steps: - *pr-labels commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) GROUP="Source Build" \ ALLOW_FAIL="false" \ BUILD="$${BUILD}" \ @@ -62,7 +63,8 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) if [[ $${BUILD} == 1 ]]; then GROUP="Source Tests (Allow Fail)" \ ALLOW_FAIL="true" \ @@ -85,7 +87,8 @@ steps: - *pr-labels commands: | FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) GROUP="no_GPL" \ ALLOW_FAIL="false" \ BUILD="$${BUILD}" \ diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index 2cf06bed9..31b4e4118 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -32,7 +32,8 @@ steps: # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) - BUILD=$(($${TRIGGER_BUILD} || $${FORCE})) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) if [[ $${BUILD} == 1 ]]; then # Launch `upload_*` jobs to store tarballs into S3 once tests are done bash .buildkite/utilities/arches_pipeline_upload.sh \ From 49420005f920288d8ca9159e4b7a30b8e039bf2a Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Mon, 2 Dec 2024 14:55:16 +0100 Subject: [PATCH 20/20] Revert "forward env vars to upload script" This reverts commit f300c57e54fcc90f50bd1818f817d6c6c7d7aac7. --- pipelines/main/platforms/launch_powerpc.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/main/platforms/launch_powerpc.jl b/pipelines/main/platforms/launch_powerpc.jl index 29977cafd..3d6a58ccb 100644 --- a/pipelines/main/platforms/launch_powerpc.jl +++ b/pipelines/main/platforms/launch_powerpc.jl @@ -15,7 +15,7 @@ const arches_file = ARGS[1] # .buildkite/pipelines/main/platforms/build_linux.yml const yaml_pipeline_file = ARGS[2] -const cmd = Cmd(`bash ".buildkite/utilities/arches_pipeline_upload.sh" "$(arches_file)" "$(yaml_pipeline_file)"`, env=copy(ENV)) +const cmd = `bash ".buildkite/utilities/arches_pipeline_upload.sh" "$(arches_file)" "$(yaml_pipeline_file)"` const parsed_julia_version = parse_version_file() const should_include_powerpc_builds = parsed_julia_version < v"1.12-"