From 2aa3c011d4c021b7084e2d38b71579e42934fc1f Mon Sep 17 00:00:00 2001 From: Rashmi Gottipati Date: Thu, 30 Jul 2026 08:49:44 -0400 Subject: [PATCH 1/6] Simplify Transformers tests detection using Buildkite if_changed Signed-off-by: Rashmi Gottipati --- .buildkite/gpu-tests/gpu-tests-H100.yml | 44 +++++++++++++------------ .buildkite/gpu-tests/gpu-tests-L4.yml | 44 +++++++++++++------------ 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/.buildkite/gpu-tests/gpu-tests-H100.yml b/.buildkite/gpu-tests/gpu-tests-H100.yml index 3116c485be..634176b928 100644 --- a/.buildkite/gpu-tests/gpu-tests-H100.yml +++ b/.buildkite/gpu-tests/gpu-tests-H100.yml @@ -85,27 +85,29 @@ steps: chmod +x .buildkite/gpu-tests/scripts/combine-coverage.sh .buildkite/gpu-tests/scripts/combine-coverage.sh - - label: ":mag: Detect changes for Transformers Tests" + # Transformers tests - always run on branch pushes + - label: ":mag: Transformers Tests (branch)" + if: build.pull_request == null agents: queue: RedHat-ModelOpt-Util command: | - if [ "$${BUILDKITE_PULL_REQUEST}" = "false" ]; then - echo "Push to branch — uploading transformers tests" - buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-H100.yml - else - git fetch --unshallow 2>/dev/null || true - git fetch origin "$${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" - DIFF_FILES=$$(git diff --name-only "origin/$${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD") - CHANGED=$$(echo "$${DIFF_FILES}" \ - | grep -E '^(src/|tests/|setup\.py$$|MANIFEST\.in$$|\.buildkite/gpu-tests/transformers-tests-|\.buildkite/gpu-tests/scripts/run-tests\.sh$$)' \ - | grep -vE '^tests/(e2e|lmeval|examples)/' \ - | grep -vE '\.md$$' \ - || true) - if [ -n "$${CHANGED}" ]; then - echo "Relevant changes detected:" - echo "$${CHANGED}" - buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-H100.yml - else - echo "No relevant changes — skipping transformers tests" - fi - fi + buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-H100.yml + + # Transformers tests - conditional on PRs when relevant paths changed + - label: ":mag: Transformers Tests (PR)" + if: build.pull_request != null + agents: + queue: RedHat-ModelOpt-Util + if_changed: + - "src/**" + - "tests/**" + - "!tests/e2e/**" + - "!tests/lmeval/**" + - "!tests/examples/**" + - "!**/*.md" + - "setup.py" + - "MANIFEST.in" + - ".buildkite/gpu-tests/transformers-tests-*" + - ".buildkite/gpu-tests/scripts/run-tests.sh" + command: | + buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-H100.yml diff --git a/.buildkite/gpu-tests/gpu-tests-L4.yml b/.buildkite/gpu-tests/gpu-tests-L4.yml index 14228cfce6..d2544e3e22 100644 --- a/.buildkite/gpu-tests/gpu-tests-L4.yml +++ b/.buildkite/gpu-tests/gpu-tests-L4.yml @@ -74,27 +74,29 @@ steps: chmod +x .buildkite/gpu-tests/scripts/combine-coverage.sh .buildkite/gpu-tests/scripts/combine-coverage.sh - - label: ":mag: Detect changes for Transformers Tests" + # Transformers tests - always run on branch pushes + - label: ":mag: Transformers Tests (branch)" + if: build.pull_request == null agents: queue: RedHat-ModelOpt-Util command: | - if [ "$${BUILDKITE_PULL_REQUEST}" = "false" ]; then - echo "Push to branch — uploading transformers tests" - buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-L4.yml - else - git fetch --unshallow 2>/dev/null || true - git fetch origin "$${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" - DIFF_FILES=$$(git diff --name-only "origin/$${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD") - CHANGED=$$(echo "$${DIFF_FILES}" \ - | grep -E '^(src/|tests/|setup\.py$$|MANIFEST\.in$$|\.buildkite/gpu-tests/transformers-tests-|\.buildkite/gpu-tests/scripts/run-tests\.sh$$)' \ - | grep -vE '^tests/(e2e|lmeval|examples)/' \ - | grep -vE '\.md$$' \ - || true) - if [ -n "$${CHANGED}" ]; then - echo "Relevant changes detected:" - echo "$${CHANGED}" - buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-L4.yml - else - echo "No relevant changes — skipping transformers tests" - fi - fi + buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-L4.yml + + # Transformers tests - conditional on PRs when relevant paths changed + - label: ":mag: Transformers Tests (PR)" + if: build.pull_request != null + agents: + queue: RedHat-ModelOpt-Util + if_changed: + - "src/**" + - "tests/**" + - "!tests/e2e/**" + - "!tests/lmeval/**" + - "!tests/examples/**" + - "!**/*.md" + - "setup.py" + - "MANIFEST.in" + - ".buildkite/gpu-tests/transformers-tests-*" + - ".buildkite/gpu-tests/scripts/run-tests.sh" + command: | + buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-L4.yml From e256fe860551b6d27bb5002f5d61274438114bc2 Mon Sep 17 00:00:00 2001 From: Rashmi Gottipati Date: Thu, 6 Aug 2026 00:10:10 -0400 Subject: [PATCH 2/6] address review comments Signed-off-by: Rashmi Gottipati --- .buildkite/gpu-tests/gpu-tests-H100.yml | 3 +-- .buildkite/gpu-tests/gpu-tests-L4.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/gpu-tests/gpu-tests-H100.yml b/.buildkite/gpu-tests/gpu-tests-H100.yml index 634176b928..965c2b50a2 100644 --- a/.buildkite/gpu-tests/gpu-tests-H100.yml +++ b/.buildkite/gpu-tests/gpu-tests-H100.yml @@ -107,7 +107,6 @@ steps: - "!**/*.md" - "setup.py" - "MANIFEST.in" - - ".buildkite/gpu-tests/transformers-tests-*" - - ".buildkite/gpu-tests/scripts/run-tests.sh" + - ".buildkite/gpu-tests/**" command: | buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-H100.yml diff --git a/.buildkite/gpu-tests/gpu-tests-L4.yml b/.buildkite/gpu-tests/gpu-tests-L4.yml index d2544e3e22..cdcae5b2f5 100644 --- a/.buildkite/gpu-tests/gpu-tests-L4.yml +++ b/.buildkite/gpu-tests/gpu-tests-L4.yml @@ -96,7 +96,6 @@ steps: - "!**/*.md" - "setup.py" - "MANIFEST.in" - - ".buildkite/gpu-tests/transformers-tests-*" - - ".buildkite/gpu-tests/scripts/run-tests.sh" + - ".buildkite/gpu-tests/**" command: | buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-L4.yml From ea23b1b78ef51c6c5653f16ade1b2762ca631d74 Mon Sep 17 00:00:00 2001 From: Rashmi Gottipati Date: Thu, 6 Aug 2026 22:49:09 -0400 Subject: [PATCH 3/6] Switch if_changed to use include/exclude attributes Signed-off-by: Rashmi Gottipati --- .buildkite/gpu-tests/gpu-tests-H100.yml | 20 +++++++++++--------- .buildkite/gpu-tests/gpu-tests-L4.yml | 20 +++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.buildkite/gpu-tests/gpu-tests-H100.yml b/.buildkite/gpu-tests/gpu-tests-H100.yml index 965c2b50a2..0173246fe6 100644 --- a/.buildkite/gpu-tests/gpu-tests-H100.yml +++ b/.buildkite/gpu-tests/gpu-tests-H100.yml @@ -99,14 +99,16 @@ steps: agents: queue: RedHat-ModelOpt-Util if_changed: - - "src/**" - - "tests/**" - - "!tests/e2e/**" - - "!tests/lmeval/**" - - "!tests/examples/**" - - "!**/*.md" - - "setup.py" - - "MANIFEST.in" - - ".buildkite/gpu-tests/**" + include: + - "src/**" + - "tests/**" + - "setup.py" + - "MANIFEST.in" + - ".buildkite/gpu-tests/**" + exclude: + - "tests/e2e/**" + - "tests/lmeval/**" + - "tests/examples/**" + - "**/*.md" command: | buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-H100.yml diff --git a/.buildkite/gpu-tests/gpu-tests-L4.yml b/.buildkite/gpu-tests/gpu-tests-L4.yml index cdcae5b2f5..cc86b28405 100644 --- a/.buildkite/gpu-tests/gpu-tests-L4.yml +++ b/.buildkite/gpu-tests/gpu-tests-L4.yml @@ -88,14 +88,16 @@ steps: agents: queue: RedHat-ModelOpt-Util if_changed: - - "src/**" - - "tests/**" - - "!tests/e2e/**" - - "!tests/lmeval/**" - - "!tests/examples/**" - - "!**/*.md" - - "setup.py" - - "MANIFEST.in" - - ".buildkite/gpu-tests/**" + include: + - "src/**" + - "tests/**" + - "setup.py" + - "MANIFEST.in" + - ".buildkite/gpu-tests/**" + exclude: + - "tests/e2e/**" + - "tests/lmeval/**" + - "tests/examples/**" + - "**/*.md" command: | buildkite-agent pipeline upload .buildkite/gpu-tests/transformers-tests-L4.yml From f43598c48985a617e498935f5ec61ed09acaa967 Mon Sep 17 00:00:00 2001 From: Rashmi Gottipati Date: Mon, 10 Aug 2026 11:32:07 -0400 Subject: [PATCH 4/6] Limit branch transformers tests to main branch only Signed-off-by: Rashmi Gottipati --- .buildkite/gpu-tests/gpu-tests-H100.yml | 6 +++--- .buildkite/gpu-tests/gpu-tests-L4.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/gpu-tests/gpu-tests-H100.yml b/.buildkite/gpu-tests/gpu-tests-H100.yml index 0173246fe6..19de0df7e5 100644 --- a/.buildkite/gpu-tests/gpu-tests-H100.yml +++ b/.buildkite/gpu-tests/gpu-tests-H100.yml @@ -85,9 +85,9 @@ steps: chmod +x .buildkite/gpu-tests/scripts/combine-coverage.sh .buildkite/gpu-tests/scripts/combine-coverage.sh - # Transformers tests - always run on branch pushes + # Transformers tests - run on main branch pushes - label: ":mag: Transformers Tests (branch)" - if: build.pull_request == null + if: build.pull_request.id == null && build.branch == "main" agents: queue: RedHat-ModelOpt-Util command: | @@ -95,7 +95,7 @@ steps: # Transformers tests - conditional on PRs when relevant paths changed - label: ":mag: Transformers Tests (PR)" - if: build.pull_request != null + if: build.pull_request.id != null agents: queue: RedHat-ModelOpt-Util if_changed: diff --git a/.buildkite/gpu-tests/gpu-tests-L4.yml b/.buildkite/gpu-tests/gpu-tests-L4.yml index cc86b28405..fa9e059263 100644 --- a/.buildkite/gpu-tests/gpu-tests-L4.yml +++ b/.buildkite/gpu-tests/gpu-tests-L4.yml @@ -74,9 +74,9 @@ steps: chmod +x .buildkite/gpu-tests/scripts/combine-coverage.sh .buildkite/gpu-tests/scripts/combine-coverage.sh - # Transformers tests - always run on branch pushes + # Transformers tests - run on main branch pushes - label: ":mag: Transformers Tests (branch)" - if: build.pull_request == null + if: build.pull_request.id == null && build.branch == "main" agents: queue: RedHat-ModelOpt-Util command: | @@ -84,7 +84,7 @@ steps: # Transformers tests - conditional on PRs when relevant paths changed - label: ":mag: Transformers Tests (PR)" - if: build.pull_request != null + if: build.pull_request.id != null agents: queue: RedHat-ModelOpt-Util if_changed: From 7ecdb81fa6dd15a846fd4c512038b04e679757ec Mon Sep 17 00:00:00 2001 From: Rashmi Gottipati Date: Wed, 12 Aug 2026 00:07:15 -0400 Subject: [PATCH 5/6] make the if condition also match release-* branches using Buildkite's regex operator Signed-off-by: Rashmi Gottipati --- .buildkite/gpu-tests/gpu-tests-H100.yml | 4 ++-- .buildkite/gpu-tests/gpu-tests-L4.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/gpu-tests/gpu-tests-H100.yml b/.buildkite/gpu-tests/gpu-tests-H100.yml index 19de0df7e5..3947b879e8 100644 --- a/.buildkite/gpu-tests/gpu-tests-H100.yml +++ b/.buildkite/gpu-tests/gpu-tests-H100.yml @@ -85,9 +85,9 @@ steps: chmod +x .buildkite/gpu-tests/scripts/combine-coverage.sh .buildkite/gpu-tests/scripts/combine-coverage.sh - # Transformers tests - run on main branch pushes + # Transformers tests - run on main and release branch pushes - label: ":mag: Transformers Tests (branch)" - if: build.pull_request.id == null && build.branch == "main" + if: build.pull_request.id == null && (build.branch == "main" || build.branch =~ /^release-/) agents: queue: RedHat-ModelOpt-Util command: | diff --git a/.buildkite/gpu-tests/gpu-tests-L4.yml b/.buildkite/gpu-tests/gpu-tests-L4.yml index fa9e059263..9dcbed2ee0 100644 --- a/.buildkite/gpu-tests/gpu-tests-L4.yml +++ b/.buildkite/gpu-tests/gpu-tests-L4.yml @@ -74,9 +74,9 @@ steps: chmod +x .buildkite/gpu-tests/scripts/combine-coverage.sh .buildkite/gpu-tests/scripts/combine-coverage.sh - # Transformers tests - run on main branch pushes + # Transformers tests - run on main and release branch pushes - label: ":mag: Transformers Tests (branch)" - if: build.pull_request.id == null && build.branch == "main" + if: build.pull_request.id == null && (build.branch == "main" || build.branch =~ /^release-/) agents: queue: RedHat-ModelOpt-Util command: | From 4cfbb1fc56d47892744efe5ed4ac65a0c59893b1 Mon Sep 17 00:00:00 2001 From: Rashmi Gottipati Date: Thu, 6 Aug 2026 23:08:11 -0400 Subject: [PATCH 6/6] Test: Change .buildkite/gpu-tests/ file (should trigger transformers tests) --- .buildkite/gpu-tests/scripts/run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/gpu-tests/scripts/run-tests.sh b/.buildkite/gpu-tests/scripts/run-tests.sh index 8e419200e5..e3735bbae8 100644 --- a/.buildkite/gpu-tests/scripts/run-tests.sh +++ b/.buildkite/gpu-tests/scripts/run-tests.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# Test change: This should trigger transformers tests TEST_TYPE="${1:?Usage: run-tests.sh }" echo "~~~ System info"