From 514825871199e8d38fbfe222c17da5cb95e414fd Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 2 Jul 2026 12:52:01 +1200 Subject: [PATCH 01/14] [ML] Automate minor version bump in CI pipeline Extend the ml-cpp-version-bump pipeline for WORKFLOW=minor (feature freeze): create the release branch from main via direct ref push, bump main to the derived next minor via PR (gradle.properties + .backportrc.json), then wait for DRA artifacts on main and the new release branch. Co-authored-by: Cursor --- .../job-version-bump-phase2-minor.json.py | 91 ++++++++ .../send_slack_version_bump_notification.sh | 50 +++-- dev-tools/bump_main_minor_freeze.sh | 169 +++++++++++++++ dev-tools/create_minor_branch.sh | 106 ++++++++++ .../test_job_version_bump_pipeline.py | 32 +++ .../unittest/test_version_bump_validation.py | 47 +++- .../unittest/test_wait_version_bump_dra.py | 12 +- dev-tools/update_backportrc.py | 94 ++++++++ dev-tools/validate_version_bump_params.sh | 113 ++++++++-- dev-tools/version_bump_lib.sh | 85 ++++++++ dev-tools/version_bump_upload_phase2.sh | 5 + dev-tools/version_bump_validation.py | 200 +++++++++++++++++- dev-tools/wait_version_bump_dra.py | 124 ++++++++--- 13 files changed, 1065 insertions(+), 63 deletions(-) create mode 100755 .buildkite/job-version-bump-phase2-minor.json.py create mode 100755 dev-tools/bump_main_minor_freeze.sh create mode 100755 dev-tools/create_minor_branch.sh create mode 100755 dev-tools/update_backportrc.py create mode 100755 dev-tools/version_bump_lib.sh diff --git a/.buildkite/job-version-bump-phase2-minor.json.py b/.buildkite/job-version-bump-phase2-minor.json.py new file mode 100755 index 000000000..129e9e09a --- /dev/null +++ b/.buildkite/job-version-bump-phase2-minor.json.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Phase 2 of the ml-cpp version bump pipeline for WORKFLOW=minor (uploaded by +# dev-tools/version_bump_upload_phase2.sh). + +import contextlib +import json +import os + + +WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest" + + +def main(): + wolfi_agent = { + "image": WOLFI_IMAGE, + "cpu": "250m", + "memory": "512Mi", + } + + pipeline_steps = [ + { + "group": "Minor version freeze", + "key": "minor-freeze", + "steps": [ + { + "label": "Create release branch ${BRANCH}", + "key": "create-minor-branch", + "agents": dict(wolfi_agent), + "command": [ + "dev-tools/create_minor_branch.sh", + ], + }, + { + "label": "Bump main to next minor", + "key": "bump-main-minor-freeze", + "agents": dict(wolfi_agent), + "env": { + "VERSION_BUMP_MERGE_AUTO": os.environ.get( + "VERSION_BUMP_MERGE_AUTO", "true" + ), + }, + "command": [ + "dev-tools/bump_main_minor_freeze.sh", + ], + }, + ], + }, + { + "label": "Notify :slack: — minor freeze PR needs approval", + "key": "queue-slack-notify", + "depends_on": "minor-freeze", + "command": [ + ".buildkite/pipelines/send_slack_version_bump_notification.sh", + ], + "agents": dict(wolfi_agent), + }, + { + "label": "Fetch DRA Artifacts", + "key": "fetch-dra-artifacts", + "depends_on": "queue-slack-notify", + "agents": { + **wolfi_agent, + "ephemeralStorage": "1Gi", + }, + "command": [ + "python3", + "dev-tools/wait_version_bump_dra.py", + ], + "timeout_in_minutes": 240, + "retry": { + "automatic": [{"exit_status": "*", "limit": 2}], + "manual": {"permit_on_passed": True}, + }, + }, + ] + + print(json.dumps({"steps": pipeline_steps}, indent=2)) + + +if __name__ == "__main__": + with contextlib.suppress(KeyboardInterrupt): + main() diff --git a/.buildkite/pipelines/send_slack_version_bump_notification.sh b/.buildkite/pipelines/send_slack_version_bump_notification.sh index 9105d3d41..dbe070751 100755 --- a/.buildkite/pipelines/send_slack_version_bump_notification.sh +++ b/.buildkite/pipelines/send_slack_version_bump_notification.sh @@ -36,21 +36,46 @@ fi pr_url="" changed="false" +minor_branch_created="false" +workflow="${WORKFLOW:-patch}" pr_url=$(buildkite-agent meta-data get "ml_cpp_version_bump_pr_url" 2>/dev/null || true) changed=$(buildkite-agent meta-data get "ml_cpp_version_bump_changed" 2>/dev/null || echo "false") -# Meta-data values must not contain stray whitespace (Breaks truthiness.) +minor_branch_created=$(buildkite-agent meta-data get "ml_cpp_minor_branch_created" 2>/dev/null || echo "false") +# Meta-data values must not contain stray whitespace (breaks truthiness.) pr_url=$(echo -n "${pr_url}" | tr -d '\r') changed=$(echo -n "${changed}" | tr -d '\r') +minor_branch_created=$(echo -n "${minor_branch_created}" | tr -d '\r') +workflow=$(echo -n "${workflow}" | tr -d '\r') -if [[ -z "${pr_url}" && "${changed}" != "true" ]]; then - echo "No version-bump PR opened (pr_url empty, ml_cpp_version_bump_changed=${changed}); skipping Slack notification." - exit 0 -fi - -if [[ -z "${pr_url}" && "${changed}" == "true" ]]; then - body_line="DRY RUN — no pull request URL (simulated bump)." +if [[ "${workflow}" == "minor" ]]; then + if [[ "${minor_branch_created}" != "true" && "${changed}" != "true" ]]; then + echo "Minor freeze: no branch created and no main-bump PR; skipping Slack notification." + exit 0 + fi + branch_line="Release branch \${BRANCH:-\"(unset)\"} created (or already present) at \${NEW_VERSION:-\"(unset)\"}." + if [[ -n "${pr_url}" ]]; then + pr_line="Main bump pull request (approval required): ${pr_url}" + elif [[ "${changed}" == "true" ]]; then + pr_line="DRY RUN — main bump PR simulated (no URL)." + else + pr_line="Main bump: no PR required (already at derived next minor)." + fi + slack_title="**Minor feature freeze — action may be required**" + slack_body="${branch_line} +${pr_line}" else - body_line="Pull request (approval required): ${pr_url}" + if [[ -z "${pr_url}" && "${changed}" != "true" ]]; then + echo "No version-bump PR opened (pr_url empty, ml_cpp_version_bump_changed=${changed}); skipping Slack notification." + exit 0 + fi + + if [[ -z "${pr_url}" && "${changed}" == "true" ]]; then + body_line="DRY RUN — no pull request URL (simulated bump)." + else + body_line="Pull request (approval required): ${pr_url}" + fi + slack_title="**Version bump PR — approval required**" + slack_body="${body_line}" fi ( @@ -63,8 +88,9 @@ steps: channels: - "${CHANNEL}" message: | - **Version bump PR — approval required** - ${body_line} + ${slack_title} + ${slack_body} + WORKFLOW: \${WORKFLOW:-"(unset)"} Branch: \${BUILDKITE_BRANCH} NEW_VERSION: \${NEW_VERSION:-"(unset)"} BRANCH (param): \${BRANCH:-"(unset)"} @@ -72,6 +98,6 @@ steps: DRY_RUN: \${DRY_RUN:-"(unset)"} Pipeline: \${BUILDKITE_BUILD_URL} Build: \${BUILDKITE_BUILD_NUMBER} - Please review and approve this pull request so it can merge (subject to branch protection). + Please review and approve the main bump pull request when present (subject to branch protection). EOF ) | buildkite-agent pipeline upload diff --git a/dev-tools/bump_main_minor_freeze.sh b/dev-tools/bump_main_minor_freeze.sh new file mode 100755 index 000000000..f8d84e3d7 --- /dev/null +++ b/dev-tools/bump_main_minor_freeze.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Minor feature freeze — Leg B: bump main to the next minor and update .backportrc.json. +# +# NEW_VERSION is the release-branch version (e.g. 9.5.0). main is bumped to +# MAIN_NEW_VERSION derived as minor+1 (e.g. 9.6.0). Opens a PR into main. +# +# Environment: same as dev-tools/bump_version.sh plus: +# BRANCH — new release branch name (for .backportrc.json only) +# +# Buildkite meta-data: +# ml_cpp_main_bump_changed — true when a main-bump PR was opened (or DRY_RUN simulates) +# ml_cpp_version_bump_changed — same as ml_cpp_main_bump_changed (DRA / Slack compat) +# ml_cpp_version_bump_main_new_version — MAIN_NEW_VERSION for DRA wait + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=version_bump_lib.sh +source "${SCRIPT_DIR}/version_bump_lib.sh" + +PYTHON="${PYTHON:-python3}" +VALIDATION_PY="${SCRIPT_DIR}/version_bump_validation.py" +UPDATE_BACKPORTRC_PY="${SCRIPT_DIR}/update_backportrc.py" +CREATE_PR_SH="${SCRIPT_DIR}/create_github_pull_request.sh" + +: "${NEW_VERSION:?NEW_VERSION must be set}" +: "${BRANCH:?BRANCH must be set}" + +NEW_VERSION="$(version_bump_trim_value "${NEW_VERSION}")" +BRANCH="$(version_bump_trim_value "${BRANCH}")" +DRY_RUN="${DRY_RUN:-false}" +TARGET_BRANCH="main" + +GRADLE_PROPS="gradle.properties" +BACKPORTRC=".backportrc.json" + +if [ "$DRY_RUN" = "true" ]; then + echo "=== DRY RUN MODE — will not push or create PR ===" +fi + +MAIN_NEW_VERSION=$("$PYTHON" "$VALIDATION_PY" derive-main-new-version --new "$NEW_VERSION") +version_bump_set_buildkite_meta "ml_cpp_version_bump_main_new_version" "$MAIN_NEW_VERSION" + +main_bump_topic_branch_name() { + local tb="ci/ml-cpp-minor-freeze-main-${MAIN_NEW_VERSION}" + if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then + tb="${tb}-bk${BUILDKITE_BUILD_NUMBER}" + fi + echo "$tb" +} + +version_bump_set_main_bump_changed() { + local changed="$1" + version_bump_set_buildkite_meta "ml_cpp_main_bump_changed" "$changed" + version_bump_set_buildkite_meta_changed "$changed" +} + +echo "=== Minor freeze Leg B: bump ${TARGET_BRANCH} ${NEW_VERSION} → ${MAIN_NEW_VERSION} ===" +version_bump_set_main_bump_changed false + +git fetch origin "$TARGET_BRANCH" + +current_version=$(read_elasticsearch_version_from_ref "origin/${TARGET_BRANCH}") +if [[ -z "$current_version" ]]; then + echo "ERROR: could not read elasticsearchVersion from origin/${TARGET_BRANCH}" >&2 + exit 1 +fi + +if ! "$PYTHON" "$VALIDATION_PY" validate-main-minor-bump \ + --current "$current_version" \ + --main-new-version "$MAIN_NEW_VERSION" \ + --release-branch-version "$NEW_VERSION" +then + exit 1 +fi + +topic_branch=$(main_bump_topic_branch_name) +git checkout -B "$topic_branch" "origin/${TARGET_BRANCH}" + +current_version=$(read_elasticsearch_version_from_file "$GRADLE_PROPS") + +if [[ "$current_version" != "$MAIN_NEW_VERSION" ]]; then + echo "Updating ${GRADLE_PROPS}: ${current_version} → ${MAIN_NEW_VERSION}" + sed_inplace "s/^elasticsearchVersion=.*/elasticsearchVersion=${MAIN_NEW_VERSION}/" "$GRADLE_PROPS" +fi + +if ! grep -q "^elasticsearchVersion=${MAIN_NEW_VERSION}$" "$GRADLE_PROPS"; then + echo "ERROR: version update verification failed on ${topic_branch}" >&2 + grep 'elasticsearchVersion' "$GRADLE_PROPS" >&2 + exit 1 +fi + +if ! "$PYTHON" "$UPDATE_BACKPORTRC_PY" \ + --path "$BACKPORTRC" \ + --new-release-branch "$BRANCH" \ + --main-new-version "$MAIN_NEW_VERSION" +then + exit 1 +fi + +if git diff-index --quiet HEAD --; then + echo "main already at ${MAIN_NEW_VERSION} and .backportrc.json is up to date — nothing to do" + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "false" + exit 0 +fi + +configure_git +git add "$GRADLE_PROPS" "$BACKPORTRC" +git commit -m "[ML] Bump version to ${MAIN_NEW_VERSION} (minor freeze)" + +if [ "$DRY_RUN" = "true" ]; then + echo " [DRY RUN] Would push origin ${topic_branch} and open PR into ${TARGET_BRANCH}" + version_bump_set_main_bump_changed true + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "true" + exit 0 +fi + +git push -u origin "$topic_branch" +echo " Pushed topic branch ${topic_branch}" + +repo_slug=$(github_repo_slug) || exit 1 + +pr_body="$(cat <&2 + exit 1 +fi + +release_branch_exists=false +release_branch_version="" +if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + release_branch_exists=true + git fetch origin "$BRANCH" + release_branch_version=$(read_elasticsearch_version_from_ref "origin/${BRANCH}") +fi + +if ! "$PYTHON" "$VALIDATION_PY" validate-minor-freeze \ + --main-version "$main_version" \ + --new "$NEW_VERSION" \ + --branch "$BRANCH" \ + $([[ "$release_branch_exists" == "true" ]] && echo --release-branch-exists) \ + ${release_branch_version:+--release-branch-version "$release_branch_version"} +then + exit 1 +fi + +if [[ "$release_branch_exists" == "true" ]]; then + echo "Release branch origin/${BRANCH} already exists at ${NEW_VERSION} — nothing to do" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_created" "true" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "false" + exit 0 +fi + +if [[ "$main_version" != "$NEW_VERSION" ]]; then + echo "ERROR: origin/${UPSTREAM_BRANCH} is ${main_version}, expected ${NEW_VERSION} before branching" >&2 + exit 1 +fi + +if [ "$DRY_RUN" = "true" ]; then + echo " [DRY RUN] Would push origin ${UPSTREAM_BRANCH}:${BRANCH}" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_created" "true" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "true" + exit 0 +fi + +configure_git +git push origin "${UPSTREAM_BRANCH}:refs/heads/${BRANCH}" +echo " Created origin/${BRANCH} from origin/${UPSTREAM_BRANCH}" + +git fetch origin "$BRANCH" +branch_version=$(read_elasticsearch_version_from_ref "origin/${BRANCH}") +if [[ "$branch_version" != "$NEW_VERSION" ]]; then + echo "ERROR: origin/${BRANCH} version is ${branch_version}, expected ${NEW_VERSION}" >&2 + exit 1 +fi + +version_bump_set_buildkite_meta "ml_cpp_minor_branch_created" "true" +version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "true" +echo "OK: release branch ${BRANCH} is at ${NEW_VERSION}" diff --git a/dev-tools/unittest/test_job_version_bump_pipeline.py b/dev-tools/unittest/test_job_version_bump_pipeline.py index b77d59159..92bdf77e3 100644 --- a/dev-tools/unittest/test_job_version_bump_pipeline.py +++ b/dev-tools/unittest/test_job_version_bump_pipeline.py @@ -21,6 +21,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] _PIPELINE_PHASE1 = _REPO_ROOT / ".buildkite" / "job-version-bump.json.py" _PIPELINE_PHASE2 = _REPO_ROOT / ".buildkite" / "job-version-bump-phase2.json.py" +_PIPELINE_PHASE2_MINOR = _REPO_ROOT / ".buildkite" / "job-version-bump-phase2-minor.json.py" def _run_phase1(extra_env: dict[str, str] | None = None) -> dict: @@ -37,6 +38,20 @@ def _run_phase1(extra_env: dict[str, str] | None = None) -> dict: return json.loads(out) +def _run_phase2_minor(extra_env: dict[str, str] | None = None) -> dict: + env = os.environ.copy() + env.pop("VERSION_BUMP_MERGE_AUTO", None) + if extra_env: + env.update(extra_env) + out = subprocess.check_output( + [sys.executable, str(_PIPELINE_PHASE2_MINOR)], + cwd=str(_REPO_ROOT), + env=env, + text=True, + ) + return json.loads(out) + + def _run_phase2(extra_env: dict[str, str] | None = None) -> dict: env = os.environ.copy() env.pop("VERSION_BUMP_MERGE_AUTO", None) @@ -180,3 +195,20 @@ def test_create_pr_script_requires_body() -> None: ) assert proc.returncode != 0 assert "--body" in proc.stderr + + +def test_phase2_minor_has_parallel_freeze_group() -> None: + pipeline = _run_phase2_minor() + group = pipeline["steps"][0] + assert group["key"] == "minor-freeze" + keys = [s["key"] for s in group["steps"]] + assert keys == ["create-minor-branch", "bump-main-minor-freeze"] + + +def test_phase2_minor_order_group_then_slack_then_dra() -> None: + pipeline = _run_phase2_minor() + assert _step_by_key(pipeline, "queue-slack-notify")["depends_on"] == "minor-freeze" + assert ( + _step_by_key(pipeline, "fetch-dra-artifacts")["depends_on"] + == "queue-slack-notify" + ) diff --git a/dev-tools/unittest/test_version_bump_validation.py b/dev-tools/unittest/test_version_bump_validation.py index b5c8deee9..1d5576482 100644 --- a/dev-tools/unittest/test_version_bump_validation.py +++ b/dev-tools/unittest/test_version_bump_validation.py @@ -175,14 +175,55 @@ def test_shell_skip_validation_env() -> None: assert out.returncode == 0, out.stderr + out.stdout +def test_derive_main_new_version() -> None: + assert vbu.derive_main_new_version("9.5.0") == "9.6.0" + + +def test_minor_freeze_ok() -> None: + main_new = vbu.validate_minor_freeze_params( + main_version="9.5.0", + new_version="9.5.0", + branch="9.5", + release_branch_exists=False, + release_branch_version=None, + ) + assert main_new == "9.6.0" + + +def test_minor_freeze_rejects_main_not_at_new_version() -> None: + with pytest.raises(ValueError, match="main elasticsearchVersion"): + vbu.validate_minor_freeze_params( + main_version="9.4.0", + new_version="9.5.0", + branch="9.5", + release_branch_exists=False, + release_branch_version=None, + ) + + +def test_main_minor_bump_ok() -> None: + vbu.validate_main_minor_bump( + current_version="9.5.0", + main_new_version="9.6.0", + release_branch_version="9.5.0", + ) + + +def test_main_minor_bump_noop() -> None: + vbu.validate_main_minor_bump( + current_version="9.6.0", + main_new_version="9.6.0", + release_branch_version="9.5.0", + ) + + @pytest.mark.skipif( not _VALIDATOR_SCRIPT.is_file(), reason="validate_version_bump_params.sh missing", ) -def test_shell_rejects_non_patch_workflow() -> None: - """Upstream may send WORKFLOW=minor; fail before git fetch.""" +def test_shell_rejects_unknown_workflow() -> None: env = os.environ.copy() - env["WORKFLOW"] = "minor" + env["WORKFLOW"] = "feature-freeze" env["NEW_VERSION"] = "9.5.1" env["BRANCH"] = "9.5" env.pop("SKIP_VERSION_VALIDATION", None) diff --git a/dev-tools/unittest/test_wait_version_bump_dra.py b/dev-tools/unittest/test_wait_version_bump_dra.py index c463620c4..852d2a94b 100644 --- a/dev-tools/unittest/test_wait_version_bump_dra.py +++ b/dev-tools/unittest/test_wait_version_bump_dra.py @@ -42,17 +42,25 @@ def advance_sleep(_seconds: float) -> None: nonlocal t t += float(_seconds) + 1.0 + def meta_side_effect(key: str) -> str | None: + if key == "ml_cpp_version_bump_noop": + return None + if key == "ml_cpp_version_bump_changed": + return "true" + return None + with ( patch.dict( "os.environ", { "BRANCH": "9.5", "NEW_VERSION": "9.5.1", + "WORKFLOW": "patch", "BUILDKITE": "false", }, clear=False, ), - patch.object(mod, "_meta_get", return_value="true"), + patch.object(mod, "_meta_get", side_effect=meta_side_effect), patch.object(mod, "_fetch_version", return_value=None), patch.object(mod.time, "monotonic", side_effect=fake_monotonic), patch.object(mod.time, "sleep", side_effect=advance_sleep), @@ -63,4 +71,4 @@ def advance_sleep(_seconds: float) -> None: assert mod.main() == 1 out = capsys.readouterr().out - assert "staging=None snapshot=None (still waiting)" in out + assert "still waiting: staging=None, snapshot=None" in out diff --git a/dev-tools/update_backportrc.py b/dev-tools/update_backportrc.py new file mode 100755 index 000000000..382d2145c --- /dev/null +++ b/dev-tools/update_backportrc.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +"""Update .backportrc.json for minor release feature freeze.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def update_backportrc_for_minor_freeze( + data: dict[str, Any], + *, + new_release_branch: str, + main_new_version: str, +) -> bool: + """Apply minor-freeze updates in place. Returns True if anything changed.""" + changed = False + + choices: list[str] = list(data.get("targetBranchChoices", [])) + if new_release_branch not in choices: + if "main" in choices: + insert_at = choices.index("main") + 1 + else: + insert_at = 0 + choices.insert(insert_at, new_release_branch) + data["targetBranchChoices"] = choices + changed = True + + mapping: dict[str, str] = dict(data.get("branchLabelMapping", {})) + new_main_key = f"^v{main_new_version}$" + old_main_keys = [k for k, v in mapping.items() if v == "main" and k != new_main_key] + for key in old_main_keys: + del mapping[key] + changed = True + if mapping.get(new_main_key) != "main": + mapping[new_main_key] = "main" + changed = True + data["branchLabelMapping"] = mapping + + return changed + + +def _cmd_update(args: argparse.Namespace) -> int: + path = Path(args.path) + if not path.is_file(): + print(f"ERROR: {path} not found", file=sys.stderr) + return 1 + + with path.open(encoding="utf-8") as handle: + data = json.load(handle) + + changed = update_backportrc_for_minor_freeze( + data, + new_release_branch=args.new_release_branch, + main_new_version=args.main_new_version, + ) + if not changed: + print(f"OK: {path} already configured for branch {args.new_release_branch} and main {args.main_new_version}") + return 0 + + with path.open("w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2) + handle.write("\n") + + print( + f"Updated {path}: added branch {args.new_release_branch}, " + f"main label mapping v{args.main_new_version}" + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Update .backportrc.json for minor freeze") + parser.add_argument("--path", default=".backportrc.json", help="Path to backportrc file") + parser.add_argument("--new-release-branch", required=True, help="New release branch (MAJOR.MINOR)") + parser.add_argument("--main-new-version", required=True, help="New version on main (MAJOR.MINOR.PATCH)") + args = parser.parse_args() + return _cmd_update(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev-tools/validate_version_bump_params.sh b/dev-tools/validate_version_bump_params.sh index 2742973c7..21e35aa6c 100755 --- a/dev-tools/validate_version_bump_params.sh +++ b/dev-tools/validate_version_bump_params.sh @@ -16,8 +16,7 @@ # Environment: # NEW_VERSION — required target stack version (MAJOR.MINOR.PATCH), unless skipped # BRANCH — required release branch (e.g. 9.5), unless skipped -# WORKFLOW — optional; defaults to patch. If set by upstream automation, must be -# exactly "patch" (this pipeline does not support minor bumps). +# WORKFLOW — optional; defaults to patch. Supported: patch, minor (feature freeze). # SKIP_VERSION_VALIDATION — set to "true" to skip (emergency override only) # PYTHON — interpreter (default: python3) # @@ -26,6 +25,28 @@ set -euo pipefail +# shellcheck disable=SC2034 +version_bump_trim_value() { + local s=$1 + s="${s//$'\r'/}" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +version_bump_set_buildkite_meta() { + local key="$1" + local value="$2" + if [[ "${BUILDKITE:-}" != "true" ]]; then + return 0 + fi + if ! command -v buildkite-agent >/dev/null 2>&1; then + echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ${key}=${value}" >&2 + return 0 + fi + buildkite-agent meta-data set "$key" "$value" +} + version_bump_set_noop_meta() { local noop="$1" if [[ "${BUILDKITE:-}" != "true" ]]; then @@ -53,19 +74,87 @@ fi : "${NEW_VERSION:?NEW_VERSION must be set}" : "${BRANCH:?BRANCH must be set}" -version_bump_trim_value() { - local s=$1 - s="${s//$'\r'/}" - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} NEW_VERSION="$(version_bump_trim_value "${NEW_VERSION}")" BRANCH="$(version_bump_trim_value "${BRANCH}")" WORKFLOW="${WORKFLOW:-patch}" +WORKFLOW="$(version_bump_trim_value "${WORKFLOW}")" + +if [[ "$WORKFLOW" == "minor" ]]; then + echo "=== Version bump validation (minor feature freeze) ===" + echo "WORKFLOW: ${WORKFLOW}" + echo "NEW_VERSION: ${NEW_VERSION} (expected on release branch ${BRANCH})" + echo "BRANCH: ${BRANCH} (release branch to create)" + + echo "Fetching origin/main and checking origin/${BRANCH}..." + git fetch origin main + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + git fetch origin "$BRANCH" + fi + + main_version=$( + git show origin/main:gradle.properties | grep '^elasticsearchVersion=' | head -1 | cut -d= -f2 | tr -d '[:space:]' || true + ) + if [[ -z "$main_version" ]]; then + echo "ERROR: could not read elasticsearchVersion from origin/main gradle.properties" >&2 + exit 1 + fi + echo "Current version on origin/main: ${main_version}" + + release_branch_exists=false + release_branch_version="" + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + release_branch_exists=true + release_branch_version=$( + git show "origin/${BRANCH}:gradle.properties" | grep '^elasticsearchVersion=' | head -1 | cut -d= -f2 | tr -d '[:space:]' || true + ) + echo "Release branch origin/${BRANCH} exists at version: ${release_branch_version:-unknown}" + else + echo "Release branch origin/${BRANCH} does not exist yet" + fi + + minor_validate_args=( + "$PYTHON" "$VALIDATION_PY" validate-minor-freeze + --main-version "$main_version" + --new "$NEW_VERSION" + --branch "$BRANCH" + ) + if [[ "$release_branch_exists" == "true" ]]; then + minor_validate_args+=(--release-branch-exists --release-branch-version "$release_branch_version") + fi + if ! "${minor_validate_args[@]}"; then + exit 1 + fi + + MAIN_NEW_VERSION=$("$PYTHON" "$VALIDATION_PY" derive-main-new-version --new "$NEW_VERSION") + version_bump_set_buildkite_meta "ml_cpp_version_bump_main_new_version" "$MAIN_NEW_VERSION" + echo "Derived MAIN_NEW_VERSION for main bump: ${MAIN_NEW_VERSION}" + + branch_needed=true + if [[ "$release_branch_exists" == "true" && "$release_branch_version" == "$NEW_VERSION" ]]; then + branch_needed=false + fi + version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "$([[ "$branch_needed" == "true" ]] && echo true || echo false)" + + main_bump_needed=true + main_trim=$(echo "$main_version" | tr -d '[:space:]') + main_new_trim=$(echo "$MAIN_NEW_VERSION" | tr -d '[:space:]') + if [[ "$main_trim" == "$main_new_trim" ]]; then + main_bump_needed=false + fi + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "$([[ "$main_bump_needed" == "true" ]] && echo true || echo false)" + + if [[ "$branch_needed" == "false" && "$main_bump_needed" == "false" ]]; then + version_bump_set_noop_meta true + echo "OK: release branch and main bump already complete — follow-up steps will no-op." + else + version_bump_set_noop_meta false + fi + exit 0 +fi + if [[ "$WORKFLOW" != "patch" ]]; then - echo "ERROR: WORKFLOW must be \"patch\" for this pipeline, got \"${WORKFLOW}\"" >&2 + echo "ERROR: WORKFLOW must be \"patch\" or \"minor\", got \"${WORKFLOW}\"" >&2 exit 1 fi @@ -74,9 +163,7 @@ echo "WORKFLOW: ${WORKFLOW}" echo "NEW_VERSION: ${NEW_VERSION}" echo "BRANCH: ${BRANCH}" -# Patch-only pipeline (no WORKFLOW=minor): consecutive patch on this release -# branch. Current version is read from origin/${BRANCH} by design — there is no -# minor-line bump mode in dev-tools/version_bump_validation.py or this pipeline. +# Patch workflow: consecutive patch increment on the release branch named BRANCH. echo "Fetching origin/${BRANCH}..." git fetch origin "$BRANCH" diff --git a/dev-tools/version_bump_lib.sh b/dev-tools/version_bump_lib.sh new file mode 100755 index 000000000..8332bffa4 --- /dev/null +++ b/dev-tools/version_bump_lib.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Shared helpers for ml-cpp release-eng version bump scripts. + +set -euo pipefail + +version_bump_trim_value() { + local s=$1 + s="${s//$'\r'/}" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +configure_git() { + git config user.name elasticsearchmachine + git config user.email 'infra-root+elasticsearchmachine@elastic.co' +} + +sed_inplace() { + local script=$1 + local target=$2 + local tmp + tmp=$(mktemp "${target}.sedtmp.XXXXXX") + if ! sed "${script}" "$target" >"$tmp"; then + rm -f "$tmp" + return 1 + fi + mv "$tmp" "$target" +} + +github_repo_slug() { + local url + url=$(git remote get-url origin 2>/dev/null || true) + if [[ "$url" =~ github\.com[:/]([^/]+)/([^/.]+)(\.git)?$ ]]; then + echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" + return 0 + fi + echo "ERROR: could not parse owner/repo from git remote url: ${url:-empty}" >&2 + return 1 +} + +version_bump_set_buildkite_meta() { + local key="$1" + local value="$2" + if [[ "${BUILDKITE:-}" != "true" ]]; then + return 0 + fi + if ! command -v buildkite-agent >/dev/null 2>&1; then + echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ${key}=${value}" >&2 + return 0 + fi + buildkite-agent meta-data set "$key" "$value" +} + +version_bump_set_buildkite_meta_changed() { + version_bump_set_buildkite_meta "ml_cpp_version_bump_changed" "$1" +} + +version_bump_set_pr_url_meta() { + local url="${1:-}" + if [[ -z "${url}" ]]; then + return 0 + fi + version_bump_set_buildkite_meta "ml_cpp_version_bump_pr_url" "$url" +} + +read_elasticsearch_version_from_file() { + local file=$1 + grep '^elasticsearchVersion=' "$file" | head -1 | cut -d= -f2 | tr -d '[:space:]' || true +} + +read_elasticsearch_version_from_ref() { + local ref=$1 + git show "${ref}:gradle.properties" | grep '^elasticsearchVersion=' | head -1 | cut -d= -f2 | tr -d '[:space:]' || true +} diff --git a/dev-tools/version_bump_upload_phase2.sh b/dev-tools/version_bump_upload_phase2.sh index 82be7561c..6f56cb620 100755 --- a/dev-tools/version_bump_upload_phase2.sh +++ b/dev-tools/version_bump_upload_phase2.sh @@ -42,4 +42,9 @@ if [[ "${noop}" == "true" ]]; then exit 0 fi +WORKFLOW="${WORKFLOW:-patch}" +if [[ "${WORKFLOW}" == "minor" ]]; then + exec python3 .buildkite/job-version-bump-phase2-minor.json.py | buildkite-agent pipeline upload +fi + exec python3 .buildkite/job-version-bump-phase2.json.py | buildkite-agent pipeline upload diff --git a/dev-tools/version_bump_validation.py b/dev-tools/version_bump_validation.py index 42cbda0e9..72a065b47 100644 --- a/dev-tools/version_bump_validation.py +++ b/dev-tools/version_bump_validation.py @@ -8,7 +8,12 @@ # compliance with the Elastic License 2.0 and the foregoing additional # limitation. # -"""Rules for ml-cpp patch release version bump parameters (Buildkite / release-eng). +"""Rules for ml-cpp release version bump parameters (Buildkite / release-eng). + +Patch and minor (feature freeze) workflows share parameter names from release-eng: +NEW_VERSION, BRANCH, WORKFLOW. For WORKFLOW=minor, NEW_VERSION is the version +expected on the new release branch (e.g. 9.5.0 on branch 9.5); main is bumped to +derive_main_new_version(NEW_VERSION) (e.g. 9.6.0). Used by dev-tools/validate_version_bump_params.sh and unit-tested under dev-tools/unittest/. @@ -115,6 +120,118 @@ def validate_version_bump_params( ) +def derive_main_new_version(release_branch_version: str) -> str: + """Return the main-branch version after minor feature freeze (minor + 1, patch 0).""" + parsed = parse_semver(release_branch_version) + if parsed is None: + raise ValueError( + f"release branch version must be MAJOR.MINOR.PATCH, got {release_branch_version!r}" + ) + major, minor, patch = parsed + if patch != 0: + raise ValueError( + "minor freeze expects NEW_VERSION patch 0 " + f"(got {release_branch_version!r})" + ) + return f"{major}.{minor + 1}.0" + + +def validate_minor_freeze_params( + *, + main_version: str, + new_version: str, + branch: str, + release_branch_exists: bool, + release_branch_version: str | None, +) -> str: + """Validate minor freeze inputs. Returns MAIN_NEW_VERSION (derived). + + NEW_VERSION is the version expected on the new release branch (BRANCH). + main must currently be at NEW_VERSION before the freeze bump. + """ + _reject_outer_whitespace("NEW_VERSION", new_version) + _reject_outer_whitespace("BRANCH", branch) + _reject_outer_whitespace("main_version", main_version) + + new_t = parse_semver(new_version) + if new_t is None: + raise ValueError( + f"NEW_VERSION must be MAJOR.MINOR.PATCH (digits only), got {new_version!r}" + ) + new_major, new_minor, new_patch = new_t + if new_patch != 0: + raise ValueError( + f"minor freeze NEW_VERSION must be X.Y.0 (patch 0), got {new_version!r}" + ) + + br = parse_release_branch(branch) + if br is None: + raise ValueError( + f"BRANCH must be MAJOR.MINOR (e.g. 9.5), got {branch!r}" + ) + br_major, br_minor = br + if br_major != new_major or br_minor != new_minor: + raise ValueError( + f"BRANCH {branch!r} must match MAJOR.MINOR of NEW_VERSION " + f"({new_major}.{new_minor}), got NEW_VERSION {new_version!r}" + ) + + main_t = parse_semver(main_version) + if main_t is None: + raise ValueError( + "elasticsearchVersion on main must be MAJOR.MINOR.PATCH, " + f"got {main_version!r}" + ) + if main_version != new_version: + raise ValueError( + "minor freeze requires main elasticsearchVersion to equal NEW_VERSION " + f"before branching (main={main_version!r}, NEW_VERSION={new_version!r})" + ) + + main_new_version = derive_main_new_version(new_version) + + if release_branch_exists: + if release_branch_version is None: + raise ValueError( + f"release branch {branch!r} exists but version could not be read" + ) + if release_branch_version != new_version: + raise ValueError( + f"release branch {branch!r} exists with version {release_branch_version!r}, " + f"expected {new_version!r}" + ) + + return main_new_version + + +def validate_main_minor_bump( + *, + current_version: str, + main_new_version: str, + release_branch_version: str, +) -> None: + """Validate bumping main from release-branch version to MAIN_NEW_VERSION.""" + _reject_outer_whitespace("current_version", current_version) + _reject_outer_whitespace("main_new_version", main_new_version) + _reject_outer_whitespace("release_branch_version", release_branch_version) + + if current_version == main_new_version: + return + + if current_version != release_branch_version: + raise ValueError( + "main bump expects current main version to equal NEW_VERSION " + f"({release_branch_version!r}), got {current_version!r}" + ) + + expected = derive_main_new_version(release_branch_version) + if main_new_version != expected: + raise ValueError( + f"MAIN_NEW_VERSION must be {expected!r} for NEW_VERSION " + f"{release_branch_version!r}, got {main_new_version!r}" + ) + + def _cmd_validate(args: argparse.Namespace) -> int: try: validate_version_bump_params( @@ -165,6 +282,87 @@ def main() -> int: p_rep.add_argument("--branch", required=True) p_rep.set_defaults(func=_cmd_validate_and_report) + p_minor = sub.add_parser( + "validate-minor-freeze", + help="check main/new/branch for WORKFLOW=minor", + ) + p_minor.add_argument("--main-version", required=True) + p_minor.add_argument("--new", required=True, dest="new") + p_minor.add_argument("--branch", required=True) + p_minor.add_argument( + "--release-branch-exists", + action="store_true", + help="origin/BRANCH already exists", + ) + p_minor.add_argument( + "--release-branch-version", + default="", + help="elasticsearchVersion on origin/BRANCH when it exists", + ) + + def _cmd_validate_minor(args_ns: argparse.Namespace) -> int: + try: + rb_ver = args_ns.release_branch_version or None + main_new = validate_minor_freeze_params( + main_version=args_ns.main_version, + new_version=args_ns.new, + branch=args_ns.branch, + release_branch_exists=args_ns.release_branch_exists, + release_branch_version=rb_ver, + ) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + print(f"OK: minor freeze NEW_VERSION={args_ns.new} on branch {args_ns.branch}") + print(f"OK: main bump target MAIN_NEW_VERSION={main_new}") + return 0 + + p_minor.set_defaults(func=_cmd_validate_minor) + + p_main = sub.add_parser( + "validate-main-minor-bump", + help="check main bump during minor freeze", + ) + p_main.add_argument("--current", required=True) + p_main.add_argument("--main-new-version", required=True) + p_main.add_argument("--release-branch-version", required=True) + + def _cmd_validate_main_bump(args_ns: argparse.Namespace) -> int: + try: + validate_main_minor_bump( + current_version=args_ns.current, + main_new_version=args_ns.main_new_version, + release_branch_version=args_ns.release_branch_version, + ) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + if args_ns.current == args_ns.main_new_version: + print(f"OK: main already at {args_ns.main_new_version} — bump step will no-op.") + else: + print( + f"OK: main minor bump {args_ns.current} → {args_ns.main_new_version}" + ) + return 0 + + p_main.set_defaults(func=_cmd_validate_main_bump) + + p_derive = sub.add_parser( + "derive-main-new-version", + help="print MAIN_NEW_VERSION for NEW_VERSION (minor freeze)", + ) + p_derive.add_argument("--new", required=True, dest="new") + + def _cmd_derive_main(args_ns: argparse.Namespace) -> int: + try: + print(derive_main_new_version(args_ns.new)) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + return 0 + + p_derive.set_defaults(func=_cmd_derive_main) + args = parser.parse_args() return args.func(args) diff --git a/dev-tools/wait_version_bump_dra.py b/dev-tools/wait_version_bump_dra.py index 50e40ce9b..bf9d4c764 100755 --- a/dev-tools/wait_version_bump_dra.py +++ b/dev-tools/wait_version_bump_dra.py @@ -11,8 +11,14 @@ """Poll DRA staging/snapshot JSON until versions match (replaces json-watcher plugin). Buildkite step conditionals cannot use build meta-data; this script reads -ml_cpp_version_bump_changed via ``buildkite-agent meta-data get`` and exits -immediately when no PR was opened. +ml_cpp_version_bump_noop / ml_cpp_version_bump_changed via ``buildkite-agent +meta-data get`` and exits immediately when the wait is not needed. + +Patch (WORKFLOW=patch): waits for staging + snapshot on BRANCH at NEW_VERSION. + +Minor (WORKFLOW=minor): waits for three artifact sets after feature freeze: + - snapshot on main at MAIN_NEW_VERSION-SNAPSHOT + - snapshot + staging on release branch BRANCH at NEW_VERSION """ from __future__ import annotations @@ -27,7 +33,6 @@ POLL_SECONDS = 30 TIMEOUT_SECONDS = 240 * 60 -# Heartbeat in Buildkite logs every N poll iterations (even when fetches return None). PROGRESS_LOG_EVERY = 1 STAGING_TMPL = "https://artifacts-staging.elastic.co/ml-cpp/latest/{branch}.json" @@ -35,11 +40,7 @@ def _meta_get(key: str) -> str | None: - """Read Buildkite meta-data. Returns None when not on Buildkite or key is unset. - - On BUILDKITE=true, missing ``buildkite-agent`` or unexpected failures exit - non-zero so we do not silently skip the DRA wait. - """ + """Read Buildkite meta-data. Returns None when not on Buildkite or key is unset.""" if os.environ.get("BUILDKITE") != "true": return None try: @@ -100,48 +101,107 @@ def _fetch_version(url: str) -> str | None: return None +def _derive_main_new_version(release_version: str) -> str: + parts = release_version.split(".") + if len(parts) != 3: + raise ValueError(f"invalid semver: {release_version!r}") + major, minor, patch = int(parts[0]), int(parts[1]), int(parts[2]) + if patch != 0: + raise ValueError(f"expected patch 0 for minor freeze, got {release_version!r}") + return f"{major}.{minor + 1}.0" + + +def _wait_for_checks(checks: list[tuple[str, str, str]]) -> int: + """Poll until all (label, url, expected_version) match.""" + print(f"Waiting for DRA artifacts (timeout {TIMEOUT_SECONDS}s, poll {POLL_SECONDS}s)...") + for label, url, expected in checks: + print(f" {label}: {expected!r} <= {url}") + + deadline = time.monotonic() + TIMEOUT_SECONDS + iteration = 0 + while time.monotonic() < deadline: + iteration += 1 + pending = [] + for label, url, expected in checks: + got = _fetch_version(url) + if got != expected: + pending.append(f"{label}={got!r}") + if not pending: + print("OK: all DRA artifact versions matched.") + return 0 + if iteration % PROGRESS_LOG_EVERY == 0: + print(f" still waiting: {', '.join(pending)}") + time.sleep(POLL_SECONDS) + + print("ERROR: timed out waiting for DRA artifact versions.", file=sys.stderr) + return 1 + + +def _wait_patch(branch: str, new_version: str) -> int: + staging_url = STAGING_TMPL.format(branch=branch) + snapshot_url = SNAPSHOT_TMPL.format(branch=branch) + checks = [ + ("staging", staging_url, new_version), + ("snapshot", snapshot_url, f"{new_version}-SNAPSHOT"), + ] + return _wait_for_checks(checks) + + +def _wait_minor(branch: str, new_version: str, main_new_version: str) -> int: + main_snapshot_url = SNAPSHOT_TMPL.format(branch="main") + branch_staging_url = STAGING_TMPL.format(branch=branch) + branch_snapshot_url = SNAPSHOT_TMPL.format(branch=branch) + checks = [ + ("main snapshot", main_snapshot_url, f"{main_new_version}-SNAPSHOT"), + ("release snapshot", branch_snapshot_url, f"{new_version}-SNAPSHOT"), + ("release staging", branch_staging_url, new_version), + ] + return _wait_for_checks(checks) + + def main() -> int: if os.environ.get("DRY_RUN") == "true": print("DRY_RUN=true — skipping DRA wait.") return 0 - if _meta_get("ml_cpp_version_bump_changed") != "true": + if _meta_get("ml_cpp_version_bump_noop") == "true": print( - "ml_cpp_version_bump_changed is not true — no PR opened; skipping DRA wait.", + "ml_cpp_version_bump_noop is true — nothing to wait for; skipping DRA wait.", file=sys.stderr, ) return 0 + workflow = os.environ.get("WORKFLOW", "patch").strip().lower() branch = os.environ.get("BRANCH", "").strip() new_version = os.environ.get("NEW_VERSION", "").strip() if not branch or not new_version: print("ERROR: BRANCH and NEW_VERSION must be set.", file=sys.stderr) return 1 - staging_url = STAGING_TMPL.format(branch=branch) - snapshot_url = SNAPSHOT_TMPL.format(branch=branch) - want_staging = new_version - want_snapshot = f"{new_version}-SNAPSHOT" - - print(f"Waiting for DRA artifacts (timeout {TIMEOUT_SECONDS}s, poll {POLL_SECONDS}s)...") - print(f" staging: {want_staging!r} <= {staging_url}") - print(f" snapshot: {want_snapshot!r} <= {snapshot_url}") + if workflow == "minor": + main_new_version = _meta_get("ml_cpp_version_bump_main_new_version") + if not main_new_version: + main_new_version = os.environ.get("MAIN_NEW_VERSION", "").strip() + if not main_new_version: + try: + main_new_version = _derive_main_new_version(new_version) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + print( + f"Minor freeze DRA wait: release branch {branch} @ {new_version}, " + f"main @ {main_new_version}" + ) + return _wait_minor(branch, new_version, main_new_version) - deadline = time.monotonic() + TIMEOUT_SECONDS - iteration = 0 - while time.monotonic() < deadline: - iteration += 1 - st = _fetch_version(staging_url) - sn = _fetch_version(snapshot_url) - if st == want_staging and sn == want_snapshot: - print("OK: staging and snapshot versions matched.") - return 0 - if iteration % PROGRESS_LOG_EVERY == 0: - print(f" staging={st!r} snapshot={sn!r} (still waiting)") - time.sleep(POLL_SECONDS) + if _meta_get("ml_cpp_version_bump_changed") != "true": + print( + "ml_cpp_version_bump_changed is not true — no PR opened; skipping DRA wait.", + file=sys.stderr, + ) + return 0 - print("ERROR: timed out waiting for DRA artifact versions.", file=sys.stderr) - return 1 + return _wait_patch(branch, new_version) if __name__ == "__main__": From 8d97bff3f4909bc95f188e9f62dd21c183004344 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 2 Jul 2026 12:58:06 +1200 Subject: [PATCH 02/14] [ML] Prefer gh auth login over GITHUB_TOKEN for version bump PRs Ignore GITHUB_TOKEN so a stale shell export does not override an interactive gh login session during local version bump testing. CI continues to use VAULT_GITHUB_TOKEN when gh is not pre-authenticated. Co-authored-by: Cursor --- dev-tools/bump_version.sh | 3 ++- dev-tools/create_github_pull_request.sh | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/dev-tools/bump_version.sh b/dev-tools/bump_version.sh index a9b5d81e6..54997ba05 100755 --- a/dev-tools/bump_version.sh +++ b/dev-tools/bump_version.sh @@ -28,7 +28,8 @@ # DRY_RUN — true to skip push and PR creation # BUILDKITE_BUILD_NUMBER — appended to topic branch name for uniqueness # VERSION_BUMP_TOPIC_BRANCH — optional override for topic branch name -# GITHUB_TOKEN / VAULT_GITHUB_TOKEN / GH_TOKEN — auth for gh (CI sets Vault token) +# VAULT_GITHUB_TOKEN / GH_TOKEN — auth for gh in CI (CI sets Vault token). +# Local runs use `gh auth login`; GITHUB_TOKEN is ignored if set in the shell. # VERSION_BUMP_NO_MERGE — set to true to open PR only (no merge / auto-merge step) # VERSION_BUMP_MERGE_AUTO — true: enable GitHub auto-merge (--auto --squash); false/unset with merge: immediate squash # VERSION_BUMP_MERGE_METHOD — merge | squash | rebase (default: squash) diff --git a/dev-tools/create_github_pull_request.sh b/dev-tools/create_github_pull_request.sh index f25ccf2e2..36ac3b0aa 100755 --- a/dev-tools/create_github_pull_request.sh +++ b/dev-tools/create_github_pull_request.sh @@ -12,7 +12,9 @@ # Create a pull request (and optionally merge) using the GitHub CLI. # # Requires: gh (https://cli.github.com/) in PATH, authenticated via: -# GITHUB_TOKEN, VAULT_GITHUB_TOKEN, or GH_TOKEN +# `gh auth login` (preferred for local runs), VAULT_GITHUB_TOKEN (CI), or GH_TOKEN. +# GITHUB_TOKEN is intentionally ignored so a stale shell export does not override +# an interactive gh login session. # If gh is missing, dev-tools/ensure_github_cli.sh runs (Wolfi apk, else Linux # tarball) unless SKIP_GH_AUTO_INSTALL=true. # @@ -117,17 +119,17 @@ case "$MERGE_METHOD" in ;; esac -# gh honors GH_TOKEN; validate after CLI args so invalid flag combinations fail without secrets. -if [[ -z "${GH_TOKEN:-}" ]]; then - if [[ -n "${GITHUB_TOKEN:-}" ]]; then - export GH_TOKEN="${GITHUB_TOKEN}" - elif [[ -n "${VAULT_GITHUB_TOKEN:-}" ]]; then - export GH_TOKEN="${VAULT_GITHUB_TOKEN}" - fi +# gh prefers GH_TOKEN over `gh auth login` credentials. Unset GH_TOKEN when the CLI +# is already logged in so local testing works with `gh auth login` even if GITHUB_TOKEN +# is exported in the shell. GITHUB_TOKEN is never used as a fallback. +if gh auth status >/dev/null 2>&1; then + unset GH_TOKEN +elif [[ -z "${GH_TOKEN:-}" && -n "${VAULT_GITHUB_TOKEN:-}" ]]; then + export GH_TOKEN="${VAULT_GITHUB_TOKEN}" fi -if [[ -z "${GH_TOKEN:-}" ]]; then - echo "ERROR: Set GITHUB_TOKEN, VAULT_GITHUB_TOKEN, or GH_TOKEN for gh auth." >&2 +if ! gh auth status >/dev/null 2>&1; then + echo "ERROR: gh is not authenticated. Run \`gh auth login\` or set VAULT_GITHUB_TOKEN / GH_TOKEN." >&2 exit 1 fi From cd92b3a515b07e2e60b5a6a65cd7b2528045f540 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 2 Jul 2026 14:16:09 +1200 Subject: [PATCH 03/14] [ML] Support testing- sandbox branch names in minor version bump Allow BRANCH=testing-MAJOR.MINOR for manual Buildkite runs: version rules strip the prefix while git ops use the full ref, and main bump plus DRA wait are skipped so sandbox testing does not touch production branches. Co-authored-by: Cursor --- dev-tools/bump_main_minor_freeze.sh | 7 ++++ .../unittest/test_version_bump_validation.py | 30 +++++++++++++++++ .../unittest/test_wait_version_bump_dra.py | 19 +++++++++++ dev-tools/validate_version_bump_params.sh | 7 ++++ dev-tools/version_bump_validation.py | 33 ++++++++++++++++--- dev-tools/wait_version_bump_dra.py | 15 +++++++++ 6 files changed, 107 insertions(+), 4 deletions(-) diff --git a/dev-tools/bump_main_minor_freeze.sh b/dev-tools/bump_main_minor_freeze.sh index f8d84e3d7..606a0d4e5 100755 --- a/dev-tools/bump_main_minor_freeze.sh +++ b/dev-tools/bump_main_minor_freeze.sh @@ -48,6 +48,13 @@ if [ "$DRY_RUN" = "true" ]; then echo "=== DRY RUN MODE — will not push or create PR ===" fi +if [[ "$BRANCH" == testing-* ]]; then + echo "Sandbox branch ${BRANCH} — skipping main bump and .backportrc.json update" + version_bump_set_main_bump_changed false + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "false" + exit 0 +fi + MAIN_NEW_VERSION=$("$PYTHON" "$VALIDATION_PY" derive-main-new-version --new "$NEW_VERSION") version_bump_set_buildkite_meta "ml_cpp_version_bump_main_new_version" "$MAIN_NEW_VERSION" diff --git a/dev-tools/unittest/test_version_bump_validation.py b/dev-tools/unittest/test_version_bump_validation.py index 1d5576482..4c711a676 100644 --- a/dev-tools/unittest/test_version_bump_validation.py +++ b/dev-tools/unittest/test_version_bump_validation.py @@ -80,6 +80,25 @@ def test_patch_ok_consecutive() -> None: ) +def test_patch_ok_with_sandbox_branch_name() -> None: + vbu.validate_version_bump_params( + current_version="9.5.0", + new_version="9.5.1", + branch="testing-9.5", + ) + + +def test_release_branch_identity_strips_testing_prefix() -> None: + assert vbu.release_branch_identity("9.5") == "9.5" + assert vbu.release_branch_identity("testing-9.5") == "9.5" + assert vbu.is_sandbox_release_branch("testing-9.5") + assert not vbu.is_sandbox_release_branch("9.5") + + +def test_parse_release_branch_accepts_sandbox_prefix() -> None: + assert vbu.parse_release_branch("testing-9.5") == (9, 5) + + def test_patch_ok_noop_same_version() -> None: vbu.validate_version_bump_params( current_version="9.5.1", @@ -190,6 +209,17 @@ def test_minor_freeze_ok() -> None: assert main_new == "9.6.0" +def test_minor_freeze_ok_sandbox_branch_name() -> None: + main_new = vbu.validate_minor_freeze_params( + main_version="9.5.0", + new_version="9.5.0", + branch="testing-9.5", + release_branch_exists=False, + release_branch_version=None, + ) + assert main_new == "9.6.0" + + def test_minor_freeze_rejects_main_not_at_new_version() -> None: with pytest.raises(ValueError, match="main elasticsearchVersion"): vbu.validate_minor_freeze_params( diff --git a/dev-tools/unittest/test_wait_version_bump_dra.py b/dev-tools/unittest/test_wait_version_bump_dra.py index 852d2a94b..63b1311ac 100644 --- a/dev-tools/unittest/test_wait_version_bump_dra.py +++ b/dev-tools/unittest/test_wait_version_bump_dra.py @@ -72,3 +72,22 @@ def meta_side_effect(key: str) -> str | None: out = capsys.readouterr().out assert "still waiting: staging=None, snapshot=None" in out + + +def test_main_skips_dra_wait_for_sandbox_branch(capsys) -> None: + mod = _load_wait_module() + with patch.dict( + "os.environ", + { + "BRANCH": "testing-9.5", + "NEW_VERSION": "9.5.0", + "WORKFLOW": "minor", + "BUILDKITE": "false", + }, + clear=False, + ): + assert mod.main() == 0 + + err = capsys.readouterr().err + assert "Sandbox release branch" in err + assert "testing-9.5" in err diff --git a/dev-tools/validate_version_bump_params.sh b/dev-tools/validate_version_bump_params.sh index 21e35aa6c..4c5ed32c8 100755 --- a/dev-tools/validate_version_bump_params.sh +++ b/dev-tools/validate_version_bump_params.sh @@ -85,6 +85,9 @@ if [[ "$WORKFLOW" == "minor" ]]; then echo "WORKFLOW: ${WORKFLOW}" echo "NEW_VERSION: ${NEW_VERSION} (expected on release branch ${BRANCH})" echo "BRANCH: ${BRANCH} (release branch to create)" + if [[ "$BRANCH" == testing-* ]]; then + echo " (sandbox: version rules use identity ${BRANCH#testing-})" + fi echo "Fetching origin/main and checking origin/${BRANCH}..." git fetch origin main @@ -142,6 +145,10 @@ if [[ "$WORKFLOW" == "minor" ]]; then if [[ "$main_trim" == "$main_new_trim" ]]; then main_bump_needed=false fi + if [[ "$BRANCH" == testing-* ]]; then + main_bump_needed=false + echo "Sandbox branch ${BRANCH} — main bump will be skipped" + fi version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "$([[ "$main_bump_needed" == "true" ]] && echo true || echo false)" if [[ "$branch_needed" == "false" && "$main_bump_needed" == "false" ]]; then diff --git a/dev-tools/version_bump_validation.py b/dev-tools/version_bump_validation.py index 72a065b47..15727b3de 100644 --- a/dev-tools/version_bump_validation.py +++ b/dev-tools/version_bump_validation.py @@ -15,6 +15,9 @@ expected on the new release branch (e.g. 9.5.0 on branch 9.5); main is bumped to derive_main_new_version(NEW_VERSION) (e.g. 9.6.0). +BRANCH may be MAJOR.MINOR or a sandbox ref ``testing-MAJOR.MINOR`` (e.g. ``testing-9.5``). +Version rules strip the ``testing-`` prefix; git operations use the full ref name. + Used by dev-tools/validate_version_bump_params.sh and unit-tested under dev-tools/unittest/. @@ -38,6 +41,19 @@ SEMVER_RE = re.compile(r"^([0-9]+)\.([0-9]+)\.([0-9]+)$") BRANCH_RE = re.compile(r"^([0-9]+)\.([0-9]+)$") +SANDBOX_BRANCH_PREFIX = "testing-" + + +def is_sandbox_release_branch(branch: str) -> bool: + """True when BRANCH is a manual-test ref (testing-MAJOR.MINOR), not a production line.""" + return branch.startswith(SANDBOX_BRANCH_PREFIX) + + +def release_branch_identity(branch: str) -> str: + """Return MAJOR.MINOR identity for version rules (strip leading testing- prefix).""" + if is_sandbox_release_branch(branch): + return branch[len(SANDBOX_BRANCH_PREFIX) :] + return branch def _reject_outer_whitespace(label: str, value: str) -> None: @@ -56,7 +72,8 @@ def parse_semver(version: str) -> Optional[Tuple[int, int, int]]: def parse_release_branch(branch: str) -> Optional[Tuple[int, int]]: - m = BRANCH_RE.match(branch) + identity = release_branch_identity(branch) + m = BRANCH_RE.match(identity) if not m: return None return (int(m.group(1)), int(m.group(2))) @@ -86,7 +103,8 @@ def validate_version_bump_params( br = parse_release_branch(branch) if br is None: raise ValueError( - f"BRANCH must be MAJOR.MINOR (e.g. 9.5), got {branch!r}" + f"BRANCH must be MAJOR.MINOR (e.g. 9.5) or " + f"{SANDBOX_BRANCH_PREFIX}MAJOR.MINOR (e.g. testing-9.5), got {branch!r}" ) br_major, br_minor = br if br_major != new_major or br_minor != new_minor: @@ -167,7 +185,8 @@ def validate_minor_freeze_params( br = parse_release_branch(branch) if br is None: raise ValueError( - f"BRANCH must be MAJOR.MINOR (e.g. 9.5), got {branch!r}" + f"BRANCH must be MAJOR.MINOR (e.g. 9.5) or " + f"{SANDBOX_BRANCH_PREFIX}MAJOR.MINOR (e.g. testing-9.5), got {branch!r}" ) br_major, br_minor = br if br_major != new_major or br_minor != new_minor: @@ -270,7 +289,7 @@ def main() -> int: ) p_val.add_argument("--current", required=True, help="elasticsearchVersion on branch") p_val.add_argument("--new", required=True, dest="new", help="NEW_VERSION") - p_val.add_argument("--branch", required=True, help="BRANCH (MAJOR.MINOR)") + p_val.add_argument("--branch", required=True, help="BRANCH (MAJOR.MINOR or testing-MAJOR.MINOR)") p_val.set_defaults(func=_cmd_validate) p_rep = sub.add_parser( @@ -314,6 +333,12 @@ def _cmd_validate_minor(args_ns: argparse.Namespace) -> int: print(f"ERROR: {e}", file=sys.stderr) return 1 print(f"OK: minor freeze NEW_VERSION={args_ns.new} on branch {args_ns.branch}") + if is_sandbox_release_branch(args_ns.branch): + identity = release_branch_identity(args_ns.branch) + print( + f"OK: sandbox branch (version identity {identity!r}); " + "main bump and DRA wait are skipped in CI" + ) print(f"OK: main bump target MAIN_NEW_VERSION={main_new}") return 0 diff --git a/dev-tools/wait_version_bump_dra.py b/dev-tools/wait_version_bump_dra.py index bf9d4c764..ed968597d 100755 --- a/dev-tools/wait_version_bump_dra.py +++ b/dev-tools/wait_version_bump_dra.py @@ -30,6 +30,13 @@ import time import urllib.error import urllib.request +from pathlib import Path + +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +import version_bump_validation as vbu POLL_SECONDS = 30 TIMEOUT_SECONDS = 240 * 60 @@ -178,6 +185,14 @@ def main() -> int: print("ERROR: BRANCH and NEW_VERSION must be set.", file=sys.stderr) return 1 + if vbu.is_sandbox_release_branch(branch): + print( + f"Sandbox release branch {branch!r} — skipping DRA wait " + f"(no artifacts published for {vbu.SANDBOX_BRANCH_PREFIX}* refs).", + file=sys.stderr, + ) + return 0 + if workflow == "minor": main_new_version = _meta_get("ml_cpp_version_bump_main_new_version") if not main_new_version: From 4193bf8e599bf1c96f266065e5589e2d107b009b Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 2 Jul 2026 14:28:23 +1200 Subject: [PATCH 04/14] [ML] Fix minor version bump uploading patch phase-2 pipeline twice Replace exec-in-pipeline with if/else so WORKFLOW=minor only uploads the minor follow-up steps once, avoiding duplicate Buildkite step keys. Co-authored-by: Cursor --- .../test_version_bump_upload_phase2.py | 100 ++++++++++++++++++ dev-tools/version_bump_upload_phase2.sh | 9 +- 2 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 dev-tools/unittest/test_version_bump_upload_phase2.py diff --git a/dev-tools/unittest/test_version_bump_upload_phase2.py b/dev-tools/unittest/test_version_bump_upload_phase2.py new file mode 100644 index 000000000..a9b5fb456 --- /dev/null +++ b/dev-tools/unittest/test_version_bump_upload_phase2.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Tests for dev-tools/version_bump_upload_phase2.sh.""" + +from __future__ import annotations + +import json +import os +import subprocess +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_UPLOAD_SCRIPT = _REPO_ROOT / "dev-tools" / "version_bump_upload_phase2.sh" + + +@pytest.fixture +def fake_buildkite_agent(tmp_path: Path) -> tuple[Path, Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + count_file = tmp_path / "upload_count" + count_file.write_text("0") + agent = bin_dir / "buildkite-agent" + agent.write_text( + textwrap.dedent( + f"""\ + #!/bin/bash + set -euo pipefail + if [[ "$1" == "meta-data" && "$2" == "get" ]]; then + echo "false" + exit 0 + fi + if [[ "$1" == "pipeline" && "$2" == "upload" ]]; then + n=$(cat "{count_file}") + echo $((n + 1)) > "{count_file}" + cat > "{tmp_path}/upload-${{n}}.json" + exit 0 + fi + echo "unexpected: $*" >&2 + exit 1 + """ + ) + ) + agent.chmod(0o755) + return bin_dir, count_file + + +def test_minor_workflow_uploads_phase2_once(fake_buildkite_agent: tuple[Path, Path]) -> None: + """WORKFLOW=minor must not also upload the patch phase-2 pipeline.""" + bin_dir, count_file = fake_buildkite_agent + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" + env["WORKFLOW"] = "minor" + env.pop("DRY_RUN", None) + + proc = subprocess.run( + ["/bin/bash", str(_UPLOAD_SCRIPT)], + cwd=str(_REPO_ROOT), + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == 0, proc.stderr + proc.stdout + assert count_file.read_text().strip() == "1" + + pipeline = json.loads((count_file.parent / "upload-0.json").read_text()) + assert pipeline["steps"][0]["key"] == "minor-freeze" + + +def test_patch_workflow_uploads_patch_phase2_once(fake_buildkite_agent: tuple[Path, Path]) -> None: + bin_dir, count_file = fake_buildkite_agent + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" + env["WORKFLOW"] = "patch" + env.pop("DRY_RUN", None) + + proc = subprocess.run( + ["/bin/bash", str(_UPLOAD_SCRIPT)], + cwd=str(_REPO_ROOT), + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == 0, proc.stderr + proc.stdout + assert count_file.read_text().strip() == "1" + + pipeline = json.loads((count_file.parent / "upload-0.json").read_text()) + assert pipeline["steps"][0]["key"] == "bump-version" diff --git a/dev-tools/version_bump_upload_phase2.sh b/dev-tools/version_bump_upload_phase2.sh index 6f56cb620..7996db367 100755 --- a/dev-tools/version_bump_upload_phase2.sh +++ b/dev-tools/version_bump_upload_phase2.sh @@ -43,8 +43,11 @@ if [[ "${noop}" == "true" ]]; then fi WORKFLOW="${WORKFLOW:-patch}" +# Do not use `exec cmd | buildkite-agent pipeline upload`: in bash, exec applies inside +# the pipeline subshell only, so the script would continue and upload a second phase-2 +# pipeline (duplicate step keys such as queue-slack-notify). if [[ "${WORKFLOW}" == "minor" ]]; then - exec python3 .buildkite/job-version-bump-phase2-minor.json.py | buildkite-agent pipeline upload + python3 .buildkite/job-version-bump-phase2-minor.json.py | buildkite-agent pipeline upload +else + python3 .buildkite/job-version-bump-phase2.json.py | buildkite-agent pipeline upload fi - -exec python3 .buildkite/job-version-bump-phase2.json.py | buildkite-agent pipeline upload From 96c07cb910cdcaef63585304c3e891ac21a369dd Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 2 Jul 2026 14:44:01 +1200 Subject: [PATCH 05/14] [ML] Fix sandbox main-bump skip calling helper before definition Define version_bump_set_main_bump_changed before the testing-* early exit so WORKFLOW=minor sandbox runs exit 0 instead of command-not-found. Co-authored-by: Cursor --- dev-tools/bump_main_minor_freeze.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev-tools/bump_main_minor_freeze.sh b/dev-tools/bump_main_minor_freeze.sh index 606a0d4e5..f8a9a1e00 100755 --- a/dev-tools/bump_main_minor_freeze.sh +++ b/dev-tools/bump_main_minor_freeze.sh @@ -48,6 +48,12 @@ if [ "$DRY_RUN" = "true" ]; then echo "=== DRY RUN MODE — will not push or create PR ===" fi +version_bump_set_main_bump_changed() { + local changed="$1" + version_bump_set_buildkite_meta "ml_cpp_main_bump_changed" "$changed" + version_bump_set_buildkite_meta_changed "$changed" +} + if [[ "$BRANCH" == testing-* ]]; then echo "Sandbox branch ${BRANCH} — skipping main bump and .backportrc.json update" version_bump_set_main_bump_changed false @@ -66,12 +72,6 @@ main_bump_topic_branch_name() { echo "$tb" } -version_bump_set_main_bump_changed() { - local changed="$1" - version_bump_set_buildkite_meta "ml_cpp_main_bump_changed" "$changed" - version_bump_set_buildkite_meta_changed "$changed" -} - echo "=== Minor freeze Leg B: bump ${TARGET_BRANCH} ${NEW_VERSION} → ${MAIN_NEW_VERSION} ===" version_bump_set_main_bump_changed false From c54ba186dc82c44fa53fcc4df0b10484f896defb Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 2 Jul 2026 15:00:06 +1200 Subject: [PATCH 06/14] [ML] Fix minor freeze Slack notify YAML for multi-line body Emit branch and PR lines as separate indented heredoc lines so the uploaded notify pipeline parses when main bump is skipped on sandbox runs. Co-authored-by: Cursor --- .../send_slack_version_bump_notification.sh | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipelines/send_slack_version_bump_notification.sh b/.buildkite/pipelines/send_slack_version_bump_notification.sh index dbe070751..cd41e2a98 100755 --- a/.buildkite/pipelines/send_slack_version_bump_notification.sh +++ b/.buildkite/pipelines/send_slack_version_bump_notification.sh @@ -58,11 +58,34 @@ if [[ "${workflow}" == "minor" ]]; then elif [[ "${changed}" == "true" ]]; then pr_line="DRY RUN — main bump PR simulated (no URL)." else - pr_line="Main bump: no PR required (already at derived next minor)." + pr_line="Main bump — no PR required (already at derived next minor)." fi slack_title="**Minor feature freeze — action may be required**" - slack_body="${branch_line} -${pr_line}" + ( + cat < Date: Thu, 2 Jul 2026 15:21:22 +1200 Subject: [PATCH 07/14] [ML] Run DRA wait script as one Buildkite command line Buildkite executes each command array element as a separate shell line; splitting python3 and the script path started an interactive REPL and blocked the fetch-dra-artifacts step until timeout. Co-authored-by: Cursor --- .buildkite/job-version-bump-phase2-minor.json.py | 3 +-- .buildkite/job-version-bump-phase2.json.py | 3 +-- dev-tools/unittest/test_job_version_bump_pipeline.py | 9 ++++++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.buildkite/job-version-bump-phase2-minor.json.py b/.buildkite/job-version-bump-phase2-minor.json.py index 129e9e09a..bb414fce5 100755 --- a/.buildkite/job-version-bump-phase2-minor.json.py +++ b/.buildkite/job-version-bump-phase2-minor.json.py @@ -72,8 +72,7 @@ def main(): "ephemeralStorage": "1Gi", }, "command": [ - "python3", - "dev-tools/wait_version_bump_dra.py", + "python3 dev-tools/wait_version_bump_dra.py", ], "timeout_in_minutes": 240, "retry": { diff --git a/.buildkite/job-version-bump-phase2.json.py b/.buildkite/job-version-bump-phase2.json.py index a05f2664a..6d35ef73c 100755 --- a/.buildkite/job-version-bump-phase2.json.py +++ b/.buildkite/job-version-bump-phase2.json.py @@ -64,8 +64,7 @@ def main(): "ephemeralStorage": "1Gi", }, "command": [ - "python3", - "dev-tools/wait_version_bump_dra.py", + "python3 dev-tools/wait_version_bump_dra.py", ], "timeout_in_minutes": 240, "retry": { diff --git a/dev-tools/unittest/test_job_version_bump_pipeline.py b/dev-tools/unittest/test_job_version_bump_pipeline.py index 92bdf77e3..e278812f3 100644 --- a/dev-tools/unittest/test_job_version_bump_pipeline.py +++ b/dev-tools/unittest/test_job_version_bump_pipeline.py @@ -119,7 +119,7 @@ def test_phase2_dra_uses_wait_script_not_meta_in_if() -> None: dra = _step_by_key(pipeline, "fetch-dra-artifacts") assert "if" not in dra assert "plugins" not in dra - assert dra["command"] == ["python3", "dev-tools/wait_version_bump_dra.py"] + assert dra["command"] == ["python3 dev-tools/wait_version_bump_dra.py"] def test_phase2_order_bump_then_slack_then_dra() -> None: @@ -212,3 +212,10 @@ def test_phase2_minor_order_group_then_slack_then_dra() -> None: _step_by_key(pipeline, "fetch-dra-artifacts")["depends_on"] == "queue-slack-notify" ) + + +def test_phase2_minor_dra_command_is_single_shell_line() -> None: + """Buildkite runs each command array element as a separate shell line.""" + pipeline = _run_phase2_minor() + dra = _step_by_key(pipeline, "fetch-dra-artifacts") + assert dra["command"] == ["python3 dev-tools/wait_version_bump_dra.py"] From 5d76bcaf5a2f917949309948420b0be785ca55d6 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 3 Jul 2026 10:01:43 +1200 Subject: [PATCH 08/14] [ML] Remove orphaned send_version_bump_notification.sh Nothing uploads this script since #3030 moved version-bump Slack notify to send_slack_version_bump_notification.sh in phase 2. Co-authored-by: Cursor --- .../send_version_bump_notification.sh | 36 ------------------- 1 file changed, 36 deletions(-) delete mode 100755 .buildkite/pipelines/send_version_bump_notification.sh diff --git a/.buildkite/pipelines/send_version_bump_notification.sh b/.buildkite/pipelines/send_version_bump_notification.sh deleted file mode 100755 index 163b46102..000000000 --- a/.buildkite/pipelines/send_version_bump_notification.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the following additional limitation. Functionality enabled by the -# files subject to the Elastic License 2.0 may only be used in production when -# invoked by an Elasticsearch process with a license key installed that permits -# use of machine learning features. You may not use this file except in -# compliance with the Elastic License 2.0 and the foregoing additional -# limitation. -# -# Slack notifications for the version bump pipeline. -# Sends notifications on build completion and when the build is blocked. -cat < Date: Fri, 3 Jul 2026 11:40:57 +1200 Subject: [PATCH 09/14] [ML] Consolidate version bump shell helpers in version_bump_lib.sh Remove duplicated helpers from bump_version.sh and validate_version_bump_params.sh so all shared functions live in one place. Co-authored-by: Cursor --- dev-tools/bump_version.sh | 74 ++--------------------- dev-tools/validate_version_bump_params.sh | 37 +----------- dev-tools/version_bump_lib.sh | 5 ++ 3 files changed, 12 insertions(+), 104 deletions(-) diff --git a/dev-tools/bump_version.sh b/dev-tools/bump_version.sh index 54997ba05..d6a49fa80 100755 --- a/dev-tools/bump_version.sh +++ b/dev-tools/bump_version.sh @@ -47,6 +47,9 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=version_bump_lib.sh +source "${SCRIPT_DIR}/version_bump_lib.sh" + PYTHON="${PYTHON:-python3}" VALIDATION_PY="${SCRIPT_DIR}/version_bump_validation.py" CREATE_PR_SH="${SCRIPT_DIR}/create_github_pull_request.sh" @@ -55,13 +58,6 @@ CREATE_PR_SH="${SCRIPT_DIR}/create_github_pull_request.sh" : "${BRANCH:?BRANCH must be set}" # Normalise env (Buildkite / Windows agents may inject trailing CR or spaces). -version_bump_trim_value() { - local s=$1 - s="${s//$'\r'/}" - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} NEW_VERSION="$(version_bump_trim_value "${NEW_VERSION}")" BRANCH="$(version_bump_trim_value "${BRANCH}")" @@ -73,18 +69,6 @@ if [ "$DRY_RUN" = "true" ]; then echo "=== DRY RUN MODE — will not push or create PR ===" fi -# Parse elastic/ml-cpp from origin (https://github.com/elastic/ml-cpp.git or git@...) -github_repo_slug() { - local url - url=$(git remote get-url origin 2>/dev/null || true) - if [[ "$url" =~ github\.com[:/]([^/]+)/([^/.]+)(\.git)?$ ]]; then - echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" - return 0 - fi - echo "ERROR: could not parse owner/repo from git remote url: ${url:-empty}" >&2 - return 1 -} - topic_branch_name() { local tb if [[ -n "${VERSION_BUMP_TOPIC_BRANCH:-}" ]]; then @@ -98,54 +82,6 @@ topic_branch_name() { echo "$tb" } -# In-place edit without GNU/BSD/BusyBox `sed -i` differences: write to a temp file then replace. -sed_inplace() { - local script=$1 - local target=$2 - local tmp - tmp=$(mktemp "${target}.sedtmp.XXXXXX") - if ! sed "${script}" "$target" >"$tmp"; then - rm -f "$tmp" - return 1 - fi - mv "$tmp" "$target" -} - -configure_git() { - git config user.name elasticsearchmachine - git config user.email 'infra-root+elasticsearchmachine@elastic.co' -} - -# Record whether this run actually opened a version-bump PR (for Buildkite DRA wait gating). -version_bump_set_buildkite_meta_changed() { - local changed="$1" - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ml_cpp_version_bump_changed=${changed}" >&2 - return 0 - fi - buildkite-agent meta-data set "ml_cpp_version_bump_changed" "$changed" -} - -# PR URL for the Slack step (after bump). Omit calling when there is no URL — Buildkite -# rejects meta-data set with an empty value ("value cannot be empty…"). -version_bump_set_pr_url_meta() { - local url="${1:-}" - if [[ -z "${url}" ]]; then - return 0 - fi - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ml_cpp_version_bump_pr_url" >&2 - return 0 - fi - buildkite-agent meta-data set "ml_cpp_version_bump_pr_url" "$url" -} - bump_version_via_pr() { local target_branch="$1" local target_version="$2" @@ -161,9 +97,7 @@ bump_version_via_pr() { # Topic branch starts at release-branch tip (same tree validation uses). git checkout -B "$topic_branch" "origin/${target_branch}" - current_version=$( - grep '^elasticsearchVersion=' "$GRADLE_PROPS" | head -1 | cut -d= -f2 | tr -d '[:space:]' || true - ) + current_version=$(read_elasticsearch_version_from_file "$GRADLE_PROPS") if [[ -z "$current_version" ]]; then echo "ERROR: could not read elasticsearchVersion from ${GRADLE_PROPS} on origin/${target_branch}" >&2 exit 1 diff --git a/dev-tools/validate_version_bump_params.sh b/dev-tools/validate_version_bump_params.sh index 4c5ed32c8..a5b9e9113 100755 --- a/dev-tools/validate_version_bump_params.sh +++ b/dev-tools/validate_version_bump_params.sh @@ -25,41 +25,10 @@ set -euo pipefail -# shellcheck disable=SC2034 -version_bump_trim_value() { - local s=$1 - s="${s//$'\r'/}" - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} - -version_bump_set_buildkite_meta() { - local key="$1" - local value="$2" - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ${key}=${value}" >&2 - return 0 - fi - buildkite-agent meta-data set "$key" "$value" -} - -version_bump_set_noop_meta() { - local noop="$1" - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ml_cpp_version_bump_noop=${noop}" >&2 - return 0 - fi - buildkite-agent meta-data set "ml_cpp_version_bump_noop" "$noop" -} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=version_bump_lib.sh +source "${SCRIPT_DIR}/version_bump_lib.sh" + PYTHON="${PYTHON:-python3}" VALIDATION_PY="${SCRIPT_DIR}/version_bump_validation.py" diff --git a/dev-tools/version_bump_lib.sh b/dev-tools/version_bump_lib.sh index 8332bffa4..32ec1c93b 100755 --- a/dev-tools/version_bump_lib.sh +++ b/dev-tools/version_bump_lib.sh @@ -66,6 +66,11 @@ version_bump_set_buildkite_meta_changed() { version_bump_set_buildkite_meta "ml_cpp_version_bump_changed" "$1" } +version_bump_set_noop_meta() { + local noop="$1" + version_bump_set_buildkite_meta "ml_cpp_version_bump_noop" "$noop" +} + version_bump_set_pr_url_meta() { local url="${1:-}" if [[ -z "${url}" ]]; then From 4299c38e16950de4a298e49d7cd99413cdefdfd0 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 3 Jul 2026 12:53:32 +1200 Subject: [PATCH 10/14] [ML] Skip ES ITs on automated version bump PRs Tag bump PRs with ci:skip-es-tests and omit Java integration test pipeline uploads when that label or version-bump topic branch names are detected. Co-authored-by: Cursor --- .buildkite/ml_pipeline/config.py | 45 ++++++ .buildkite/pipeline.json.py | 11 +- dev-tools/bump_main_minor_freeze.sh | 1 + dev-tools/bump_version.sh | 1 + dev-tools/create_github_pull_request.sh | 26 +++- dev-tools/unittest/test_ml_pipeline_config.py | 130 ++++++++++++++++++ 6 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 dev-tools/unittest/test_ml_pipeline_config.py diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 8b13ec39d..91bc4b022 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -21,6 +21,29 @@ } ) +# Skip Elasticsearch Java IT pipelines on automated version-bump PRs (metadata-only +# changes). Applied via ci:skip-es-tests label and/or version-bump topic branch names. +SKIP_ES_TESTS_LABEL = "ci:skip-es-tests" + +_VERSION_BUMP_TOPIC_BRANCH_PATTERNS = ( + re.compile(r"^ci/ml-cpp-version-bump-"), + re.compile(r"^ci/ml-cpp-minor-freeze-main-"), +) + + +def normalize_buildkite_branch(branch: str) -> str: + """Return the PR source branch name from BUILDKITE_BRANCH (fork or same-repo).""" + + if ":" in branch: + branch = branch.split(":", 1)[1] + # pull-requests.json uses buildkite_branch_name_separator: "+" + return branch.replace("+", "/") + + +def is_version_bump_topic_branch(branch: str) -> bool: + normalized = normalize_buildkite_branch(branch) + return any(pattern.search(normalized) for pattern in _VERSION_BUMP_TOPIC_BRANCH_PATTERNS) + class Config: build_windows: bool = False @@ -32,6 +55,7 @@ class Config: run_pytorch_tests: bool = False run_serverless_tests: bool = False deploy_serverless_qa: bool = False + skip_es_tests: bool = False action: str = "build" def parse_comment(self): @@ -198,6 +222,27 @@ def parse(self): self.build_x86_64 = "--build-x86_64" self.run_qa_tests = False + self._apply_skip_es_tests() + + def _apply_skip_es_tests(self): + """Skip Java ES IT pipelines for automated version-bump PRs.""" + + if self.skip_es_tests: + return + + for env_key in ("GITHUB_PR_LABELS", "BUILDKITE_PULL_REQUEST_LABELS"): + raw = os.environ.get(env_key, "") + if not raw: + continue + labels = [label.strip().lower() for label in raw.split(",")] + if SKIP_ES_TESTS_LABEL in labels: + self.skip_es_tests = True + return + + branch = os.environ.get("BUILDKITE_BRANCH", "") + if branch and is_version_bump_topic_branch(branch): + self.skip_es_tests = True + def _apply_serverless_kv_from_comment(self): """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ.""" diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index eff889d8a..812d6ce33 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -75,10 +75,11 @@ def main(): pipeline_steps.append(build_linux) if config.build_x86_64: - pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline", - ".buildkite/pipelines/run_es_tests_x86_64.yml.sh")) - pipeline_steps.append(pipeline_steps.generate_step("Upload ES inference tests x86_64 runner pipeline", - ".buildkite/pipelines/run_es_inference_tests_x86_64.yml.sh")) + if not config.skip_es_tests: + pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline", + ".buildkite/pipelines/run_es_tests_x86_64.yml.sh")) + pipeline_steps.append(pipeline_steps.generate_step("Upload ES inference tests x86_64 runner pipeline", + ".buildkite/pipelines/run_es_inference_tests_x86_64.yml.sh")) # We only use linux x86_64 builds for QA tests. if config.run_qa_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline", @@ -86,7 +87,7 @@ def main(): if config.run_pytorch_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline", ".buildkite/pipelines/run_pytorch_tests.yml.sh")) - if config.build_aarch64: + if config.build_aarch64 and not config.skip_es_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) diff --git a/dev-tools/bump_main_minor_freeze.sh b/dev-tools/bump_main_minor_freeze.sh index f8a9a1e00..a7ee1fca6 100755 --- a/dev-tools/bump_main_minor_freeze.sh +++ b/dev-tools/bump_main_minor_freeze.sh @@ -157,6 +157,7 @@ local -a pr_cmd=( --head "$topic_branch" --title "[ML] Bump version to ${MAIN_NEW_VERSION} (minor freeze)" --body "$pr_body" + --label "ci:skip-es-tests" ) if [[ "${VERSION_BUMP_NO_MERGE:-}" != "true" ]]; then if [[ "${VERSION_BUMP_MERGE_AUTO:-}" == "true" ]]; then diff --git a/dev-tools/bump_version.sh b/dev-tools/bump_version.sh index d6a49fa80..720340fbe 100755 --- a/dev-tools/bump_version.sh +++ b/dev-tools/bump_version.sh @@ -166,6 +166,7 @@ EOF --head "$topic_branch" --title "[ML] Bump version to ${target_version}" --body "$pr_body" + --label "ci:skip-es-tests" ) if [[ "${VERSION_BUMP_NO_MERGE:-}" != "true" ]]; then if [[ "${VERSION_BUMP_MERGE_AUTO:-}" == "true" ]]; then diff --git a/dev-tools/create_github_pull_request.sh b/dev-tools/create_github_pull_request.sh index 36ac3b0aa..00d6d16f0 100755 --- a/dev-tools/create_github_pull_request.sh +++ b/dev-tools/create_github_pull_request.sh @@ -20,7 +20,7 @@ # # Usage: # create_github_pull_request.sh --repo ORG/REPO --base BASE --head HEAD \ -# --title T --body B [--merge | --merge-auto] [--merge-method merge|squash|rebase] +# --title T --body B [--label NAME] [--merge | --merge-auto] [--merge-method merge|squash|rebase] # # On success, prints the PR URL to stdout (single line). Merge progress to stderr. # @@ -57,6 +57,7 @@ BODY="" DO_MERGE="false" DO_MERGE_AUTO="false" MERGE_METHOD="${VERSION_BUMP_MERGE_METHOD:-squash}" +LABELS=() while [[ $# -gt 0 ]]; do case "$1" in @@ -80,6 +81,10 @@ while [[ $# -gt 0 ]]; do BODY="$2" shift 2 ;; + --label) + LABELS+=("$2") + shift 2 + ;; --merge) DO_MERGE="true" shift 1 @@ -133,12 +138,19 @@ if ! gh auth status >/dev/null 2>&1; then exit 1 fi -PR_URL=$(gh pr create \ - --repo "$REPO" \ - --base "$BASE" \ - --head "$HEAD_REF" \ - --title "$TITLE" \ - --body "$BODY") +declare -a create_cmd=( + gh pr create + --repo "$REPO" + --base "$BASE" + --head "$HEAD_REF" + --title "$TITLE" + --body "$BODY" +) +for label in "${LABELS[@]}"; do + create_cmd+=(--label "$label") +done + +PR_URL=$("${create_cmd[@]}") echo "$PR_URL" diff --git a/dev-tools/unittest/test_ml_pipeline_config.py b/dev-tools/unittest/test_ml_pipeline_config.py new file mode 100644 index 000000000..c845a04d5 --- /dev/null +++ b/dev-tools/unittest/test_ml_pipeline_config.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Tests for .buildkite/ml_pipeline/config.py (PR pipeline gating).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_BUILDKITE_DIR = _REPO_ROOT / ".buildkite" +_PIPELINE_JSON = _BUILDKITE_DIR / "pipeline.json.py" + +sys.path.insert(0, str(_BUILDKITE_DIR)) +import ml_pipeline.config as pipeline_config # noqa: E402 + + +@pytest.fixture(autouse=True) +def _clear_skip_es_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ( + "GITHUB_PR_LABELS", + "BUILDKITE_PULL_REQUEST_LABELS", + "BUILDKITE_BRANCH", + "GITHUB_PR_TRIGGER_COMMENT", + ): + monkeypatch.delenv(key, raising=False) + + +def test_normalize_buildkite_branch_fork_and_plus_separator() -> None: + assert ( + pipeline_config.normalize_buildkite_branch("edsavage:ci+ml-cpp-version-bump-9.5-9.5.1") + == "ci/ml-cpp-version-bump-9.5-9.5.1" + ) + assert ( + pipeline_config.normalize_buildkite_branch("ci/ml-cpp-version-bump-9.5-9.5.1") + == "ci/ml-cpp-version-bump-9.5-9.5.1" + ) + + +@pytest.mark.parametrize( + "branch", + [ + "ci/ml-cpp-version-bump-9.5-9.5.1", + "edsavage:ci/ml-cpp-minor-freeze-main-9.6.0-bk42", + "ci+ml-cpp-minor-freeze-main-9.6.0", + ], +) +def test_is_version_bump_topic_branch_matches_automation_branches(branch: str) -> None: + assert pipeline_config.is_version_bump_topic_branch(branch) + + +def test_is_version_bump_topic_branch_rejects_feature_branches() -> None: + assert not pipeline_config.is_version_bump_topic_branch("feature/minor-version-bump") + + +def test_skip_es_tests_from_label(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_PR_LABELS", ":ml,ci:skip-es-tests") + config = pipeline_config.Config() + config.parse() + assert config.skip_es_tests is True + + +def test_skip_es_tests_from_topic_branch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv( + "BUILDKITE_BRANCH", + "edsavage:ci/ml-cpp-version-bump-9.5-9.5.1-bk99", + ) + config = pipeline_config.Config() + config.parse() + assert config.skip_es_tests is True + + +def test_skip_es_tests_false_for_normal_pr(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_PR_LABELS", ":ml,>enhancement") + monkeypatch.setenv("BUILDKITE_BRANCH", "edsavage:feature/my-change") + config = pipeline_config.Config() + config.parse() + assert config.skip_es_tests is False + + +def test_pipeline_json_omits_es_test_upload_steps_when_skip_set( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GITHUB_PR_LABELS", "ci:skip-es-tests") + env = os.environ.copy() + proc = subprocess.run( + [sys.executable, str(_PIPELINE_JSON)], + check=True, + capture_output=True, + text=True, + cwd=str(_REPO_ROOT), + env=env, + ) + pipeline = json.loads(proc.stdout) + labels = [step.get("label", "") for step in pipeline["steps"]] + assert not any("ES tests" in label for label in labels) + assert not any("Inference Integration Tests" in label for label in labels) + + +def test_pipeline_json_includes_es_test_upload_steps_by_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GITHUB_PR_LABELS", raising=False) + monkeypatch.delenv("BUILDKITE_BRANCH", raising=False) + env = {k: v for k, v in os.environ.items() if k != "GITHUB_PR_TRIGGER_COMMENT"} + proc = subprocess.run( + [sys.executable, str(_PIPELINE_JSON)], + check=True, + capture_output=True, + text=True, + cwd=str(_REPO_ROOT), + env=env, + ) + pipeline = json.loads(proc.stdout) + labels = [step.get("label", "") for step in pipeline["steps"]] + assert any("ES tests x86_64" in label for label in labels) + assert any("ES tests aarch64" in label for label in labels) From d5ff3f5b8b3b04e0cbbc8c4a0c52ead0f4c8d5a1 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 3 Jul 2026 13:29:25 +1200 Subject: [PATCH 11/14] [ML] Fix skip-es-tests branch detection for fork PR BUILDKITE_BRANCH Buildkite uses author+branch/with/slashes for fork PRs; only split on the first '+' when '/' is present. Also consult GITHUB_PR_BRANCH from the PR bot. Co-authored-by: Cursor --- .buildkite/ml_pipeline/config.py | 18 +++++++++++++----- dev-tools/unittest/test_ml_pipeline_config.py | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 91bc4b022..3363335aa 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -36,8 +36,14 @@ def normalize_buildkite_branch(branch: str) -> str: if ":" in branch: branch = branch.split(":", 1)[1] - # pull-requests.json uses buildkite_branch_name_separator: "+" - return branch.replace("+", "/") + if "+" in branch: + if "/" in branch: + # Fork PR: author+branch/with/slashes (only the author separator is "+"). + branch = branch.split("+", 1)[1] + else: + # Branch name with "/" encoded as "+" throughout (no fork author prefix). + branch = branch.replace("+", "/") + return branch def is_version_bump_topic_branch(branch: str) -> bool: @@ -239,9 +245,11 @@ def _apply_skip_es_tests(self): self.skip_es_tests = True return - branch = os.environ.get("BUILDKITE_BRANCH", "") - if branch and is_version_bump_topic_branch(branch): - self.skip_es_tests = True + for env_key in ("GITHUB_PR_BRANCH", "BUILDKITE_BRANCH"): + branch = os.environ.get(env_key, "") + if branch and is_version_bump_topic_branch(branch): + self.skip_es_tests = True + return def _apply_serverless_kv_from_comment(self): """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ.""" diff --git a/dev-tools/unittest/test_ml_pipeline_config.py b/dev-tools/unittest/test_ml_pipeline_config.py index c845a04d5..4ecf2ffcc 100644 --- a/dev-tools/unittest/test_ml_pipeline_config.py +++ b/dev-tools/unittest/test_ml_pipeline_config.py @@ -44,6 +44,16 @@ def test_normalize_buildkite_branch_fork_and_plus_separator() -> None: pipeline_config.normalize_buildkite_branch("edsavage:ci+ml-cpp-version-bump-9.5-9.5.1") == "ci/ml-cpp-version-bump-9.5-9.5.1" ) + assert ( + pipeline_config.normalize_buildkite_branch( + "edsavage+ci/ml-cpp-version-bump-manual-test-9.5.0" + ) + == "ci/ml-cpp-version-bump-manual-test-9.5.0" + ) + assert ( + pipeline_config.normalize_buildkite_branch("ci+ml-cpp-minor-freeze-main-9.6.0") + == "ci/ml-cpp-minor-freeze-main-9.6.0" + ) assert ( pipeline_config.normalize_buildkite_branch("ci/ml-cpp-version-bump-9.5-9.5.1") == "ci/ml-cpp-version-bump-9.5-9.5.1" @@ -76,13 +86,20 @@ def test_skip_es_tests_from_label(monkeypatch: pytest.MonkeyPatch) -> None: def test_skip_es_tests_from_topic_branch(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv( "BUILDKITE_BRANCH", - "edsavage:ci/ml-cpp-version-bump-9.5-9.5.1-bk99", + "edsavage+ci/ml-cpp-version-bump-manual-test-9.5.0", ) config = pipeline_config.Config() config.parse() assert config.skip_es_tests is True +def test_skip_es_tests_from_github_pr_branch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_PR_BRANCH", "ci/ml-cpp-version-bump-manual-test-9.5.0") + config = pipeline_config.Config() + config.parse() + assert config.skip_es_tests is True + + def test_skip_es_tests_false_for_normal_pr(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GITHUB_PR_LABELS", ":ml,>enhancement") monkeypatch.setenv("BUILDKITE_BRANCH", "edsavage:feature/my-change") From 99166903ba0aadd6daa18c9b2e79c42d16c081ef Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 3 Jul 2026 13:36:01 +1200 Subject: [PATCH 12/14] [ML] Isolate pipeline config tests from Buildkite PR branch env Subprocess tests must not inherit GITHUB_PR_BRANCH/BUILDKITE_BRANCH from the agent when asserting default ES IT scheduling behaviour. Co-authored-by: Cursor --- dev-tools/unittest/test_ml_pipeline_config.py | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/dev-tools/unittest/test_ml_pipeline_config.py b/dev-tools/unittest/test_ml_pipeline_config.py index 4ecf2ffcc..59d9d83ef 100644 --- a/dev-tools/unittest/test_ml_pipeline_config.py +++ b/dev-tools/unittest/test_ml_pipeline_config.py @@ -27,15 +27,26 @@ sys.path.insert(0, str(_BUILDKITE_DIR)) import ml_pipeline.config as pipeline_config # noqa: E402 +_PR_BRANCH_ENV_KEYS = ( + "GITHUB_PR_LABELS", + "BUILDKITE_PULL_REQUEST_LABELS", + "BUILDKITE_BRANCH", + "GITHUB_PR_BRANCH", + "GITHUB_PR_TRIGGER_COMMENT", +) + + +def _subprocess_env(**overrides: str) -> dict[str, str]: + env = os.environ.copy() + for key in _PR_BRANCH_ENV_KEYS: + env.pop(key, None) + env.update(overrides) + return env + @pytest.fixture(autouse=True) def _clear_skip_es_env(monkeypatch: pytest.MonkeyPatch) -> None: - for key in ( - "GITHUB_PR_LABELS", - "BUILDKITE_PULL_REQUEST_LABELS", - "BUILDKITE_BRANCH", - "GITHUB_PR_TRIGGER_COMMENT", - ): + for key in _PR_BRANCH_ENV_KEYS: monkeypatch.delenv(key, raising=False) @@ -111,15 +122,13 @@ def test_skip_es_tests_false_for_normal_pr(monkeypatch: pytest.MonkeyPatch) -> N def test_pipeline_json_omits_es_test_upload_steps_when_skip_set( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("GITHUB_PR_LABELS", "ci:skip-es-tests") - env = os.environ.copy() proc = subprocess.run( [sys.executable, str(_PIPELINE_JSON)], check=True, capture_output=True, text=True, cwd=str(_REPO_ROOT), - env=env, + env=_subprocess_env(GITHUB_PR_LABELS="ci:skip-es-tests"), ) pipeline = json.loads(proc.stdout) labels = [step.get("label", "") for step in pipeline["steps"]] @@ -127,19 +136,14 @@ def test_pipeline_json_omits_es_test_upload_steps_when_skip_set( assert not any("Inference Integration Tests" in label for label in labels) -def test_pipeline_json_includes_es_test_upload_steps_by_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.delenv("GITHUB_PR_LABELS", raising=False) - monkeypatch.delenv("BUILDKITE_BRANCH", raising=False) - env = {k: v for k, v in os.environ.items() if k != "GITHUB_PR_TRIGGER_COMMENT"} +def test_pipeline_json_includes_es_test_upload_steps_by_default() -> None: proc = subprocess.run( [sys.executable, str(_PIPELINE_JSON)], check=True, capture_output=True, text=True, cwd=str(_REPO_ROOT), - env=env, + env=_subprocess_env(), ) pipeline = json.loads(proc.stdout) labels = [step.get("label", "") for step in pipeline["steps"]] From 6baeb66cc487d77101f589391d4916d3f16276e3 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 3 Jul 2026 14:56:41 +1200 Subject: [PATCH 13/14] [ML] Skip Linux x86_64 debug PR CI for version bump PRs Reuse ci:skip-es-tests / version-bump branch detection to omit the extra Linux debug build and test steps on metadata-only bump PRs. Co-authored-by: Cursor --- .buildkite/ml_pipeline/config.py | 39 +++++++++++++------ .buildkite/pipeline.json.py | 4 +- .buildkite/pipelines/build_linux.json.py | 13 ++++++- dev-tools/unittest/test_ml_pipeline_config.py | 35 +++++++++++++++++ 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 3363335aa..2f93bb742 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -21,9 +21,12 @@ } ) -# Skip Elasticsearch Java IT pipelines on automated version-bump PRs (metadata-only -# changes). Applied via ci:skip-es-tests label and/or version-bump topic branch names. -SKIP_ES_TESTS_LABEL = "ci:skip-es-tests" +# Trim PR CI for automated version-bump PRs (metadata-only changes): skip Java ES IT +# pipelines and the extra Linux x86_64 debug build/test pair. Applied via +# ci:skip-es-tests label and/or version-bump topic branch names. +SKIP_VERSION_BUMP_PR_CI_LABEL = "ci:skip-es-tests" +# Backward-compatible alias for callers/tests that reference the old name. +SKIP_ES_TESTS_LABEL = SKIP_VERSION_BUMP_PR_CI_LABEL _VERSION_BUMP_TOPIC_BRANCH_PATTERNS = ( re.compile(r"^ci/ml-cpp-version-bump-"), @@ -61,9 +64,15 @@ class Config: run_pytorch_tests: bool = False run_serverless_tests: bool = False deploy_serverless_qa: bool = False - skip_es_tests: bool = False + skip_version_bump_pr_ci: bool = False action: str = "build" + @property + def skip_es_tests(self) -> bool: + """Backward-compatible alias for skip_version_bump_pr_ci.""" + + return self.skip_version_bump_pr_ci + def parse_comment(self): """ Parse environment variables set from GitHub PR comments @@ -228,12 +237,12 @@ def parse(self): self.build_x86_64 = "--build-x86_64" self.run_qa_tests = False - self._apply_skip_es_tests() + self._apply_skip_version_bump_pr_ci() - def _apply_skip_es_tests(self): - """Skip Java ES IT pipelines for automated version-bump PRs.""" + def _apply_skip_version_bump_pr_ci(self): + """Skip extra PR CI (Java ES ITs, Linux x86_64 debug) for version-bump PRs.""" - if self.skip_es_tests: + if self.skip_version_bump_pr_ci: return for env_key in ("GITHUB_PR_LABELS", "BUILDKITE_PULL_REQUEST_LABELS"): @@ -241,16 +250,24 @@ def _apply_skip_es_tests(self): if not raw: continue labels = [label.strip().lower() for label in raw.split(",")] - if SKIP_ES_TESTS_LABEL in labels: - self.skip_es_tests = True + if SKIP_VERSION_BUMP_PR_CI_LABEL in labels: + self.skip_version_bump_pr_ci = True return for env_key in ("GITHUB_PR_BRANCH", "BUILDKITE_BRANCH"): branch = os.environ.get(env_key, "") if branch and is_version_bump_topic_branch(branch): - self.skip_es_tests = True + self.skip_version_bump_pr_ci = True return + +def should_skip_version_bump_pr_ci() -> bool: + """Return True when PR CI should omit Java ITs and Linux x86_64 debug steps.""" + + config = Config() + config.parse() + return config.skip_version_bump_pr_ci + def _apply_serverless_kv_from_comment(self): """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ.""" diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 812d6ce33..13b22c2df 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -75,7 +75,7 @@ def main(): pipeline_steps.append(build_linux) if config.build_x86_64: - if not config.skip_es_tests: + if not config.skip_version_bump_pr_ci: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline", ".buildkite/pipelines/run_es_tests_x86_64.yml.sh")) pipeline_steps.append(pipeline_steps.generate_step("Upload ES inference tests x86_64 runner pipeline", @@ -87,7 +87,7 @@ def main(): if config.run_pytorch_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline", ".buildkite/pipelines/run_pytorch_tests.yml.sh")) - if config.build_aarch64 and not config.skip_es_tests: + if config.build_aarch64 and not config.skip_version_bump_pr_ci: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py index 1ce32b606..913e8e154 100755 --- a/.buildkite/pipelines/build_linux.json.py +++ b/.buildkite/pipelines/build_linux.json.py @@ -17,9 +17,19 @@ import argparse import json import os +import sys from itertools import product + +def should_skip_version_bump_pr_ci() -> bool: + buildkite_dir = os.path.join(os.path.dirname(__file__), "..") + if buildkite_dir not in sys.path: + sys.path.insert(0, buildkite_dir) + from ml_pipeline.config import should_skip_version_bump_pr_ci as _should_skip + + return _should_skip() + archs = [ "x86_64", "aarch64", @@ -210,7 +220,8 @@ def main(args): # Add debug build/test steps for PR builds to detect compilation errors with optimization disabled if os.environ.get("BUILDKITE_PIPELINE_SLUG", "ml-cpp-pr-builds") != "ml-cpp-debug-build" and \ - os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false": + os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and \ + not should_skip_version_bump_pr_ci(): debug_build_key = "build_test_linux-x86_64-RelWithDebInfo-debug" pipeline_steps.append({ diff --git a/dev-tools/unittest/test_ml_pipeline_config.py b/dev-tools/unittest/test_ml_pipeline_config.py index 59d9d83ef..a16838e0a 100644 --- a/dev-tools/unittest/test_ml_pipeline_config.py +++ b/dev-tools/unittest/test_ml_pipeline_config.py @@ -23,6 +23,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] _BUILDKITE_DIR = _REPO_ROOT / ".buildkite" _PIPELINE_JSON = _BUILDKITE_DIR / "pipeline.json.py" +_BUILD_LINUX_JSON = _BUILDKITE_DIR / "pipelines" / "build_linux.json.py" sys.path.insert(0, str(_BUILDKITE_DIR)) import ml_pipeline.config as pipeline_config # noqa: E402 @@ -149,3 +150,37 @@ def test_pipeline_json_includes_es_test_upload_steps_by_default() -> None: labels = [step.get("label", "") for step in pipeline["steps"]] assert any("ES tests x86_64" in label for label in labels) assert any("ES tests aarch64" in label for label in labels) + + +def _run_build_linux_pipeline(**env_overrides: str) -> list[str]: + env = _subprocess_env( + BUILDKITE_PULL_REQUEST="123", + BUILDKITE_PIPELINE_SLUG="ml-cpp-pr-builds", + **env_overrides, + ) + proc = subprocess.run( + [ + sys.executable, + str(_BUILD_LINUX_JSON), + "--action=build", + "--build-aarch64", + "--build-x86_64", + ], + check=True, + capture_output=True, + text=True, + cwd=str(_REPO_ROOT), + env=env, + ) + pipeline = json.loads(proc.stdout) + return [step.get("label", "") for step in pipeline["steps"]] + + +def test_build_linux_includes_debug_steps_for_normal_pr() -> None: + labels = _run_build_linux_pipeline() + assert any("RelWithDebInfo (debug)" in label for label in labels) + + +def test_build_linux_omits_debug_steps_when_version_bump_skip_set() -> None: + labels = _run_build_linux_pipeline(GITHUB_PR_LABELS="ci:skip-es-tests") + assert not any("RelWithDebInfo (debug)" in label for label in labels) From f4b677fb25657db997fb07da11ca6142034131f9 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 3 Jul 2026 15:28:48 +1200 Subject: [PATCH 14/14] [ML] Address Copilot review comments on version bump PR Fix Config._apply_serverless_kv_from_comment indentation, remove invalid top-level local in bump_main_minor_freeze.sh, and build validate-minor-freeze args with a proper array in create_minor_branch.sh. Co-authored-by: Cursor --- .buildkite/ml_pipeline/config.py | 16 ++++++++-------- dev-tools/bump_main_minor_freeze.sh | 2 +- dev-tools/create_minor_branch.sh | 16 ++++++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 2f93bb742..194a87ac1 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -260,14 +260,6 @@ def _apply_skip_version_bump_pr_ci(self): self.skip_version_bump_pr_ci = True return - -def should_skip_version_bump_pr_ci() -> bool: - """Return True when PR CI should omit Java ITs and Linux x86_64 debug steps.""" - - config = Config() - config.parse() - return config.skip_version_bump_pr_ci - def _apply_serverless_kv_from_comment(self): """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ.""" @@ -289,3 +281,11 @@ def _apply_serverless_kv_from_comment(self): continue os.environ[key] = value + +def should_skip_version_bump_pr_ci() -> bool: + """Return True when PR CI should omit Java ITs and Linux x86_64 debug steps.""" + + config = Config() + config.parse() + return config.skip_version_bump_pr_ci + diff --git a/dev-tools/bump_main_minor_freeze.sh b/dev-tools/bump_main_minor_freeze.sh index a7ee1fca6..9886cdd2b 100755 --- a/dev-tools/bump_main_minor_freeze.sh +++ b/dev-tools/bump_main_minor_freeze.sh @@ -150,7 +150,7 @@ When merging is enabled (\`VERSION_BUMP_NO_MERGE\` not true): **auto-merge** if EOF )" -local -a pr_cmd=( +pr_cmd=( "$CREATE_PR_SH" --repo "$repo_slug" --base "$TARGET_BRANCH" diff --git a/dev-tools/create_minor_branch.sh b/dev-tools/create_minor_branch.sh index 980b2ea87..bdce49e25 100755 --- a/dev-tools/create_minor_branch.sh +++ b/dev-tools/create_minor_branch.sh @@ -61,12 +61,16 @@ if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then release_branch_version=$(read_elasticsearch_version_from_ref "origin/${BRANCH}") fi -if ! "$PYTHON" "$VALIDATION_PY" validate-minor-freeze \ - --main-version "$main_version" \ - --new "$NEW_VERSION" \ - --branch "$BRANCH" \ - $([[ "$release_branch_exists" == "true" ]] && echo --release-branch-exists) \ - ${release_branch_version:+--release-branch-version "$release_branch_version"} +minor_validate_args=( + --main-version "$main_version" + --new "$NEW_VERSION" + --branch "$BRANCH" +) +if [[ "$release_branch_exists" == "true" ]]; then + minor_validate_args+=(--release-branch-exists --release-branch-version "$release_branch_version") +fi + +if ! "$PYTHON" "$VALIDATION_PY" validate-minor-freeze "${minor_validate_args[@]}" then exit 1 fi