[ML] Automate minor version bump in CI pipeline

.buildkite/job-version-bump-phase2-minor.json.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+# Phase 2 of the ml-cpp version bump pipeline for WORKFLOW=minor (uploaded by
+# dev-tools/version_bump_upload_phase2.sh).
+
+import contextlib
+import json
+import os
+
+
+WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest"
+
+
+def main():
+    wolfi_agent = {
+        "image": WOLFI_IMAGE,
+        "cpu": "250m",
+        "memory": "512Mi",
+    }
+
+    pipeline_steps = [
+        {
+            "group": "Minor version freeze",
+            "key": "minor-freeze",
+            "steps": [
+                {
+                    "label": "Create release branch ${BRANCH}",
+                    "key": "create-minor-branch",
+                    "agents": dict(wolfi_agent),
+                    "command": [
+                        "dev-tools/create_minor_branch.sh",
+                    ],
+                },
+                {
+                    "label": "Bump main to next minor",
+                    "key": "bump-main-minor-freeze",
+                    "agents": dict(wolfi_agent),
+                    "env": {
+                        "VERSION_BUMP_MERGE_AUTO": os.environ.get(
+                            "VERSION_BUMP_MERGE_AUTO", "true"
+                        ),
+                    },
+                    "command": [
+                        "dev-tools/bump_main_minor_freeze.sh",
+                    ],
+                },
+            ],
+        },
+        {
+            "label": "Notify :slack: — minor freeze PR needs approval",
+            "key": "queue-slack-notify",
+            "depends_on": "minor-freeze",
+            "command": [
+                ".buildkite/pipelines/send_slack_version_bump_notification.sh",
+            ],
+            "agents": dict(wolfi_agent),
+        },
+        {
+            "label": "Fetch DRA Artifacts",
+            "key": "fetch-dra-artifacts",
+            "depends_on": "queue-slack-notify",
+            "agents": {
+                **wolfi_agent,
+                "ephemeralStorage": "1Gi",
+            },
+            "command": [
+                "python3 dev-tools/wait_version_bump_dra.py",
+            ],
+            "timeout_in_minutes": 240,
+            "retry": {
+                "automatic": [{"exit_status": "*", "limit": 2}],
+                "manual": {"permit_on_passed": True},
+            },
+        },
+    ]
+
+    print(json.dumps({"steps": pipeline_steps}, indent=2))
+
+
+if __name__ == "__main__":
+    with contextlib.suppress(KeyboardInterrupt):
+        main()

.buildkite/job-version-bump-phase2.json.py

@@ -64,8 +64,7 @@ def main():
                 "ephemeralStorage": "1Gi",
             },
             "command": [
-                "python3",
-                "dev-tools/wait_version_bump_dra.py",
+                "python3 dev-tools/wait_version_bump_dra.py",
             ],
             "timeout_in_minutes": 240,
             "retry": {

.buildkite/ml_pipeline/config.py

@@ -21,6 +21,38 @@
     }
 )
 
+# Trim PR CI for automated version-bump PRs (metadata-only changes): skip Java ES IT
+# pipelines and the extra Linux x86_64 debug build/test pair. Applied via
+# ci:skip-es-tests label and/or version-bump topic branch names.
+SKIP_VERSION_BUMP_PR_CI_LABEL = "ci:skip-es-tests"
+# Backward-compatible alias for callers/tests that reference the old name.
+SKIP_ES_TESTS_LABEL = SKIP_VERSION_BUMP_PR_CI_LABEL
+
+_VERSION_BUMP_TOPIC_BRANCH_PATTERNS = (
+    re.compile(r"^ci/ml-cpp-version-bump-"),
+    re.compile(r"^ci/ml-cpp-minor-freeze-main-"),
+)
+
+
+def normalize_buildkite_branch(branch: str) -> str:
+    """Return the PR source branch name from BUILDKITE_BRANCH (fork or same-repo)."""
+
+    if ":" in branch:
+        branch = branch.split(":", 1)[1]
+    if "+" in branch:
+        if "/" in branch:
+            # Fork PR: author+branch/with/slashes (only the author separator is "+").
+            branch = branch.split("+", 1)[1]
+        else:
+            # Branch name with "/" encoded as "+" throughout (no fork author prefix).
+            branch = branch.replace("+", "/")
+    return branch
+
+
+def is_version_bump_topic_branch(branch: str) -> bool:
+    normalized = normalize_buildkite_branch(branch)
+    return any(pattern.search(normalized) for pattern in _VERSION_BUMP_TOPIC_BRANCH_PATTERNS)
+
 
 class Config:
     build_windows: bool = False
@@ -32,8 +64,15 @@ class Config:
     run_pytorch_tests: bool = False
     run_serverless_tests: bool = False
     deploy_serverless_qa: bool = False
+    skip_version_bump_pr_ci: bool = False
     action: str = "build"
 
+    @property
+    def skip_es_tests(self) -> bool:
+        """Backward-compatible alias for skip_version_bump_pr_ci."""
+
+        return self.skip_version_bump_pr_ci
+
     def parse_comment(self):
         """ Parse environment variables set from GitHub PR comments
 
@@ -198,6 +237,29 @@ def parse(self):
             self.build_x86_64 = "--build-x86_64"
             self.run_qa_tests = False
 
+        self._apply_skip_version_bump_pr_ci()
+
+    def _apply_skip_version_bump_pr_ci(self):
+        """Skip extra PR CI (Java ES ITs, Linux x86_64 debug) for version-bump PRs."""
+
+        if self.skip_version_bump_pr_ci:
+            return
+
+        for env_key in ("GITHUB_PR_LABELS", "BUILDKITE_PULL_REQUEST_LABELS"):
+            raw = os.environ.get(env_key, "")
+            if not raw:
+                continue
+            labels = [label.strip().lower() for label in raw.split(",")]
+            if SKIP_VERSION_BUMP_PR_CI_LABEL in labels:
+                self.skip_version_bump_pr_ci = True
+                return
+
+        for env_key in ("GITHUB_PR_BRANCH", "BUILDKITE_BRANCH"):
+            branch = os.environ.get(env_key, "")
+            if branch and is_version_bump_topic_branch(branch):
+                self.skip_version_bump_pr_ci = True
+                return
+
     def _apply_serverless_kv_from_comment(self):
         """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ."""
 
@@ -219,3 +281,11 @@ def _apply_serverless_kv_from_comment(self):
                 continue
             os.environ[key] = value
 
+
+def should_skip_version_bump_pr_ci() -> bool:
+    """Return True when PR CI should omit Java ITs and Linux x86_64 debug steps."""
+
+    config = Config()
+    config.parse()
+    return config.skip_version_bump_pr_ci
+

.buildkite/pipeline.json.py

@@ -75,18 +75,19 @@ def main():
         pipeline_steps.append(build_linux)
 
         if config.build_x86_64:
-            pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
-                                                               ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
-            pipeline_steps.append(pipeline_steps.generate_step("Upload ES inference tests x86_64 runner pipeline",
-                                                               ".buildkite/pipelines/run_es_inference_tests_x86_64.yml.sh"))
+            if not config.skip_version_bump_pr_ci:
+                pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
+                                                                   ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
+                pipeline_steps.append(pipeline_steps.generate_step("Upload ES inference tests x86_64 runner pipeline",
+                                                                   ".buildkite/pipelines/run_es_inference_tests_x86_64.yml.sh"))
             # We only use linux x86_64 builds for QA tests.
             if config.run_qa_tests:
                 pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline",
                                                                    ".buildkite/pipelines/run_qa_tests.yml.sh"))
             if config.run_pytorch_tests:
                 pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline",
                                                                    ".buildkite/pipelines/run_pytorch_tests.yml.sh"))
-        if config.build_aarch64:
+        if config.build_aarch64 and not config.skip_version_bump_pr_ci:
             pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline",
                                                                ".buildkite/pipelines/run_es_tests_aarch64.yml.sh"))
 

.buildkite/pipelines/build_linux.json.py

@@ -17,9 +17,19 @@
 import argparse
 import json
 import os
+import sys
 
 from itertools import product
 
+
+def should_skip_version_bump_pr_ci() -> bool:
+    buildkite_dir = os.path.join(os.path.dirname(__file__), "..")
+    if buildkite_dir not in sys.path:
+        sys.path.insert(0, buildkite_dir)
+    from ml_pipeline.config import should_skip_version_bump_pr_ci as _should_skip
+
+    return _should_skip()
+
 archs = [
     "x86_64",
     "aarch64",
@@ -210,7 +220,8 @@ def main(args):
 
     # Add debug build/test steps for PR builds to detect compilation errors with optimization disabled
     if os.environ.get("BUILDKITE_PIPELINE_SLUG", "ml-cpp-pr-builds") != "ml-cpp-debug-build" and \
-            os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false":
+            os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and \
+            not should_skip_version_bump_pr_ci():
         debug_build_key = "build_test_linux-x86_64-RelWithDebInfo-debug"
 
         pipeline_steps.append({

.buildkite/pipelines/send_slack_version_bump_notification.sh

@@ -36,21 +36,69 @@ fi
 
 pr_url=""
 changed="false"
+minor_branch_created="false"
+workflow="${WORKFLOW:-patch}"
 pr_url=$(buildkite-agent meta-data get "ml_cpp_version_bump_pr_url" 2>/dev/null || true)
 changed=$(buildkite-agent meta-data get "ml_cpp_version_bump_changed" 2>/dev/null || echo "false")
-# Meta-data values must not contain stray whitespace (Breaks truthiness.)
+minor_branch_created=$(buildkite-agent meta-data get "ml_cpp_minor_branch_created" 2>/dev/null || echo "false")
+# Meta-data values must not contain stray whitespace (breaks truthiness.)
 pr_url=$(echo -n "${pr_url}" | tr -d '\r')
 changed=$(echo -n "${changed}" | tr -d '\r')
+minor_branch_created=$(echo -n "${minor_branch_created}" | tr -d '\r')
+workflow=$(echo -n "${workflow}" | tr -d '\r')
 
-if [[ -z "${pr_url}" && "${changed}" != "true" ]]; then
-    echo "No version-bump PR opened (pr_url empty, ml_cpp_version_bump_changed=${changed}); skipping Slack notification."
+if [[ "${workflow}" == "minor" ]]; then
+    if [[ "${minor_branch_created}" != "true" && "${changed}" != "true" ]]; then
+        echo "Minor freeze: no branch created and no main-bump PR; skipping Slack notification."
+        exit 0
+    fi
+    branch_line="Release branch \${BRANCH:-\"(unset)\"} created (or already present) at \${NEW_VERSION:-\"(unset)\"}."
+    if [[ -n "${pr_url}" ]]; then
+        pr_line="Main bump pull request (approval required): ${pr_url}"
+    elif [[ "${changed}" == "true" ]]; then
+        pr_line="DRY RUN — main bump PR simulated (no URL)."
+    else
+        pr_line="Main bump — no PR required (already at derived next minor)."
+    fi
+    slack_title="**Minor feature freeze — action may be required**"
+    (
+        cat <<EOF
+steps:
+  - label: "Schedule :slack: notification (version bump)"
+    command: "echo schedule :slack: notification"
+    notify:
+      - slack:
+          channels:
+            - "${CHANNEL}"
+          message: |
+            ${slack_title}
+            ${branch_line}
+            ${pr_line}
+            WORKFLOW: \${WORKFLOW:-"(unset)"}
+            Branch: \${BUILDKITE_BRANCH}
+            NEW_VERSION: \${NEW_VERSION:-"(unset)"}
+            BRANCH (param): \${BRANCH:-"(unset)"}
+            VERSION_BUMP_MERGE_AUTO: \${VERSION_BUMP_MERGE_AUTO:-"(unset)"}
+            DRY_RUN: \${DRY_RUN:-"(unset)"}
+            Pipeline: \${BUILDKITE_BUILD_URL}
+            Build: \${BUILDKITE_BUILD_NUMBER}
+            Please review and approve the main bump pull request when present (subject to branch protection).
+EOF
+    ) | buildkite-agent pipeline upload
     exit 0
-fi
-
-if [[ -z "${pr_url}" && "${changed}" == "true" ]]; then
-    body_line="DRY RUN — no pull request URL (simulated bump)."
 else
-    body_line="Pull request (approval required): ${pr_url}"
+    if [[ -z "${pr_url}" && "${changed}" != "true" ]]; then
+        echo "No version-bump PR opened (pr_url empty, ml_cpp_version_bump_changed=${changed}); skipping Slack notification."
+        exit 0
+    fi
+
+    if [[ -z "${pr_url}" && "${changed}" == "true" ]]; then
+        body_line="DRY RUN — no pull request URL (simulated bump)."
+    else
+        body_line="Pull request (approval required): ${pr_url}"
+    fi
+    slack_title="**Version bump PR — approval required**"
+    slack_body="${body_line}"
 fi
 
 (
@@ -63,15 +111,16 @@ steps:
           channels:
             - "${CHANNEL}"
           message: |
-            **Version bump PR — approval required**
-            ${body_line}
+            ${slack_title}
+            ${slack_body}
+            WORKFLOW: \${WORKFLOW:-"(unset)"}
             Branch: \${BUILDKITE_BRANCH}
             NEW_VERSION: \${NEW_VERSION:-"(unset)"}
             BRANCH (param): \${BRANCH:-"(unset)"}
             VERSION_BUMP_MERGE_AUTO: \${VERSION_BUMP_MERGE_AUTO:-"(unset)"}
             DRY_RUN: \${DRY_RUN:-"(unset)"}
             Pipeline: \${BUILDKITE_BUILD_URL}
             Build: \${BUILDKITE_BUILD_NUMBER}
-            Please review and approve this pull request so it can merge (subject to branch protection).
+            Please review and approve the main bump pull request when present (subject to branch protection).
 EOF
 ) | buildkite-agent pipeline upload