diff --git a/.buildkite/job-version-bump-phase2-minor.json.py b/.buildkite/job-version-bump-phase2-minor.json.py new file mode 100755 index 000000000..bb414fce5 --- /dev/null +++ b/.buildkite/job-version-bump-phase2-minor.json.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Phase 2 of the ml-cpp version bump pipeline for WORKFLOW=minor (uploaded by +# dev-tools/version_bump_upload_phase2.sh). + +import contextlib +import json +import os + + +WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest" + + +def main(): + wolfi_agent = { + "image": WOLFI_IMAGE, + "cpu": "250m", + "memory": "512Mi", + } + + pipeline_steps = [ + { + "group": "Minor version freeze", + "key": "minor-freeze", + "steps": [ + { + "label": "Create release branch ${BRANCH}", + "key": "create-minor-branch", + "agents": dict(wolfi_agent), + "command": [ + "dev-tools/create_minor_branch.sh", + ], + }, + { + "label": "Bump main to next minor", + "key": "bump-main-minor-freeze", + "agents": dict(wolfi_agent), + "env": { + "VERSION_BUMP_MERGE_AUTO": os.environ.get( + "VERSION_BUMP_MERGE_AUTO", "true" + ), + }, + "command": [ + "dev-tools/bump_main_minor_freeze.sh", + ], + }, + ], + }, + { + "label": "Notify :slack: — minor freeze PR needs approval", + "key": "queue-slack-notify", + "depends_on": "minor-freeze", + "command": [ + ".buildkite/pipelines/send_slack_version_bump_notification.sh", + ], + "agents": dict(wolfi_agent), + }, + { + "label": "Fetch DRA Artifacts", + "key": "fetch-dra-artifacts", + "depends_on": "queue-slack-notify", + "agents": { + **wolfi_agent, + "ephemeralStorage": "1Gi", + }, + "command": [ + "python3 dev-tools/wait_version_bump_dra.py", + ], + "timeout_in_minutes": 240, + "retry": { + "automatic": [{"exit_status": "*", "limit": 2}], + "manual": {"permit_on_passed": True}, + }, + }, + ] + + print(json.dumps({"steps": pipeline_steps}, indent=2)) + + +if __name__ == "__main__": + with contextlib.suppress(KeyboardInterrupt): + main() diff --git a/.buildkite/job-version-bump-phase2.json.py b/.buildkite/job-version-bump-phase2.json.py index a05f2664a..6d35ef73c 100755 --- a/.buildkite/job-version-bump-phase2.json.py +++ b/.buildkite/job-version-bump-phase2.json.py @@ -64,8 +64,7 @@ def main(): "ephemeralStorage": "1Gi", }, "command": [ - "python3", - "dev-tools/wait_version_bump_dra.py", + "python3 dev-tools/wait_version_bump_dra.py", ], "timeout_in_minutes": 240, "retry": { diff --git a/.buildkite/pipelines/send_slack_version_bump_notification.sh b/.buildkite/pipelines/send_slack_version_bump_notification.sh index 9105d3d41..cd41e2a98 100755 --- a/.buildkite/pipelines/send_slack_version_bump_notification.sh +++ b/.buildkite/pipelines/send_slack_version_bump_notification.sh @@ -36,21 +36,69 @@ fi pr_url="" changed="false" +minor_branch_created="false" +workflow="${WORKFLOW:-patch}" pr_url=$(buildkite-agent meta-data get "ml_cpp_version_bump_pr_url" 2>/dev/null || true) changed=$(buildkite-agent meta-data get "ml_cpp_version_bump_changed" 2>/dev/null || echo "false") -# Meta-data values must not contain stray whitespace (Breaks truthiness.) +minor_branch_created=$(buildkite-agent meta-data get "ml_cpp_minor_branch_created" 2>/dev/null || echo "false") +# Meta-data values must not contain stray whitespace (breaks truthiness.) pr_url=$(echo -n "${pr_url}" | tr -d '\r') changed=$(echo -n "${changed}" | tr -d '\r') +minor_branch_created=$(echo -n "${minor_branch_created}" | tr -d '\r') +workflow=$(echo -n "${workflow}" | tr -d '\r') -if [[ -z "${pr_url}" && "${changed}" != "true" ]]; then - echo "No version-bump PR opened (pr_url empty, ml_cpp_version_bump_changed=${changed}); skipping Slack notification." +if [[ "${workflow}" == "minor" ]]; then + if [[ "${minor_branch_created}" != "true" && "${changed}" != "true" ]]; then + echo "Minor freeze: no branch created and no main-bump PR; skipping Slack notification." + exit 0 + fi + branch_line="Release branch \${BRANCH:-\"(unset)\"} created (or already present) at \${NEW_VERSION:-\"(unset)\"}." + if [[ -n "${pr_url}" ]]; then + pr_line="Main bump pull request (approval required): ${pr_url}" + elif [[ "${changed}" == "true" ]]; then + pr_line="DRY RUN — main bump PR simulated (no URL)." + else + pr_line="Main bump — no PR required (already at derived next minor)." + fi + slack_title="**Minor feature freeze — action may be required**" + ( + cat <&2 + exit 1 +fi + +if ! "$PYTHON" "$VALIDATION_PY" validate-main-minor-bump \ + --current "$current_version" \ + --main-new-version "$MAIN_NEW_VERSION" \ + --release-branch-version "$NEW_VERSION" +then + exit 1 +fi + +topic_branch=$(main_bump_topic_branch_name) +git checkout -B "$topic_branch" "origin/${TARGET_BRANCH}" + +current_version=$(read_elasticsearch_version_from_file "$GRADLE_PROPS") + +if [[ "$current_version" != "$MAIN_NEW_VERSION" ]]; then + echo "Updating ${GRADLE_PROPS}: ${current_version} → ${MAIN_NEW_VERSION}" + sed_inplace "s/^elasticsearchVersion=.*/elasticsearchVersion=${MAIN_NEW_VERSION}/" "$GRADLE_PROPS" +fi + +if ! grep -q "^elasticsearchVersion=${MAIN_NEW_VERSION}$" "$GRADLE_PROPS"; then + echo "ERROR: version update verification failed on ${topic_branch}" >&2 + grep 'elasticsearchVersion' "$GRADLE_PROPS" >&2 + exit 1 +fi + +if ! "$PYTHON" "$UPDATE_BACKPORTRC_PY" \ + --path "$BACKPORTRC" \ + --new-release-branch "$BRANCH" \ + --main-new-version "$MAIN_NEW_VERSION" +then + exit 1 +fi + +if git diff-index --quiet HEAD --; then + echo "main already at ${MAIN_NEW_VERSION} and .backportrc.json is up to date — nothing to do" + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "false" + exit 0 +fi + +configure_git +git add "$GRADLE_PROPS" "$BACKPORTRC" +git commit -m "[ML] Bump version to ${MAIN_NEW_VERSION} (minor freeze)" + +if [ "$DRY_RUN" = "true" ]; then + echo " [DRY RUN] Would push origin ${topic_branch} and open PR into ${TARGET_BRANCH}" + version_bump_set_main_bump_changed true + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "true" + exit 0 +fi + +git push -u origin "$topic_branch" +echo " Pushed topic branch ${topic_branch}" + +repo_slug=$(github_repo_slug) || exit 1 + +pr_body="$(cat </dev/null || true) - if [[ "$url" =~ github\.com[:/]([^/]+)/([^/.]+)(\.git)?$ ]]; then - echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" - return 0 - fi - echo "ERROR: could not parse owner/repo from git remote url: ${url:-empty}" >&2 - return 1 -} - topic_branch_name() { local tb if [[ -n "${VERSION_BUMP_TOPIC_BRANCH:-}" ]]; then @@ -97,54 +82,6 @@ topic_branch_name() { echo "$tb" } -# In-place edit without GNU/BSD/BusyBox `sed -i` differences: write to a temp file then replace. -sed_inplace() { - local script=$1 - local target=$2 - local tmp - tmp=$(mktemp "${target}.sedtmp.XXXXXX") - if ! sed "${script}" "$target" >"$tmp"; then - rm -f "$tmp" - return 1 - fi - mv "$tmp" "$target" -} - -configure_git() { - git config user.name elasticsearchmachine - git config user.email 'infra-root+elasticsearchmachine@elastic.co' -} - -# Record whether this run actually opened a version-bump PR (for Buildkite DRA wait gating). -version_bump_set_buildkite_meta_changed() { - local changed="$1" - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ml_cpp_version_bump_changed=${changed}" >&2 - return 0 - fi - buildkite-agent meta-data set "ml_cpp_version_bump_changed" "$changed" -} - -# PR URL for the Slack step (after bump). Omit calling when there is no URL — Buildkite -# rejects meta-data set with an empty value ("value cannot be empty…"). -version_bump_set_pr_url_meta() { - local url="${1:-}" - if [[ -z "${url}" ]]; then - return 0 - fi - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ml_cpp_version_bump_pr_url" >&2 - return 0 - fi - buildkite-agent meta-data set "ml_cpp_version_bump_pr_url" "$url" -} - bump_version_via_pr() { local target_branch="$1" local target_version="$2" @@ -160,9 +97,7 @@ bump_version_via_pr() { # Topic branch starts at release-branch tip (same tree validation uses). git checkout -B "$topic_branch" "origin/${target_branch}" - current_version=$( - grep '^elasticsearchVersion=' "$GRADLE_PROPS" | head -1 | cut -d= -f2 | tr -d '[:space:]' || true - ) + current_version=$(read_elasticsearch_version_from_file "$GRADLE_PROPS") if [[ -z "$current_version" ]]; then echo "ERROR: could not read elasticsearchVersion from ${GRADLE_PROPS} on origin/${target_branch}" >&2 exit 1 diff --git a/dev-tools/create_github_pull_request.sh b/dev-tools/create_github_pull_request.sh index f25ccf2e2..36ac3b0aa 100755 --- a/dev-tools/create_github_pull_request.sh +++ b/dev-tools/create_github_pull_request.sh @@ -12,7 +12,9 @@ # Create a pull request (and optionally merge) using the GitHub CLI. # # Requires: gh (https://cli.github.com/) in PATH, authenticated via: -# GITHUB_TOKEN, VAULT_GITHUB_TOKEN, or GH_TOKEN +# `gh auth login` (preferred for local runs), VAULT_GITHUB_TOKEN (CI), or GH_TOKEN. +# GITHUB_TOKEN is intentionally ignored so a stale shell export does not override +# an interactive gh login session. # If gh is missing, dev-tools/ensure_github_cli.sh runs (Wolfi apk, else Linux # tarball) unless SKIP_GH_AUTO_INSTALL=true. # @@ -117,17 +119,17 @@ case "$MERGE_METHOD" in ;; esac -# gh honors GH_TOKEN; validate after CLI args so invalid flag combinations fail without secrets. -if [[ -z "${GH_TOKEN:-}" ]]; then - if [[ -n "${GITHUB_TOKEN:-}" ]]; then - export GH_TOKEN="${GITHUB_TOKEN}" - elif [[ -n "${VAULT_GITHUB_TOKEN:-}" ]]; then - export GH_TOKEN="${VAULT_GITHUB_TOKEN}" - fi +# gh prefers GH_TOKEN over `gh auth login` credentials. Unset GH_TOKEN when the CLI +# is already logged in so local testing works with `gh auth login` even if GITHUB_TOKEN +# is exported in the shell. GITHUB_TOKEN is never used as a fallback. +if gh auth status >/dev/null 2>&1; then + unset GH_TOKEN +elif [[ -z "${GH_TOKEN:-}" && -n "${VAULT_GITHUB_TOKEN:-}" ]]; then + export GH_TOKEN="${VAULT_GITHUB_TOKEN}" fi -if [[ -z "${GH_TOKEN:-}" ]]; then - echo "ERROR: Set GITHUB_TOKEN, VAULT_GITHUB_TOKEN, or GH_TOKEN for gh auth." >&2 +if ! gh auth status >/dev/null 2>&1; then + echo "ERROR: gh is not authenticated. Run \`gh auth login\` or set VAULT_GITHUB_TOKEN / GH_TOKEN." >&2 exit 1 fi diff --git a/dev-tools/create_minor_branch.sh b/dev-tools/create_minor_branch.sh new file mode 100755 index 000000000..980b2ea87 --- /dev/null +++ b/dev-tools/create_minor_branch.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Minor feature freeze — Leg A: create release branch from main (direct ref push). +# +# NEW_VERSION is the version expected on BRANCH (e.g. 9.5.0 on 9.5). main must +# already be at NEW_VERSION; this step does not change any version file. +# +# Environment: +# NEW_VERSION, BRANCH — required (from release-eng; WORKFLOW=minor) +# DRY_RUN — true to skip push +# +# Buildkite meta-data: +# ml_cpp_minor_branch_created — true when branch was created or already OK +# ml_cpp_minor_branch_needed — false when branch already exists at NEW_VERSION + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=version_bump_lib.sh +source "${SCRIPT_DIR}/version_bump_lib.sh" + +PYTHON="${PYTHON:-python3}" +VALIDATION_PY="${SCRIPT_DIR}/version_bump_validation.py" + +: "${NEW_VERSION:?NEW_VERSION must be set}" +: "${BRANCH:?BRANCH must be set}" + +NEW_VERSION="$(version_bump_trim_value "${NEW_VERSION}")" +BRANCH="$(version_bump_trim_value "${BRANCH}")" +DRY_RUN="${DRY_RUN:-false}" +UPSTREAM_BRANCH="main" + +if [ "$DRY_RUN" = "true" ]; then + echo "=== DRY RUN MODE — will not push release branch ===" +fi + +echo "=== Minor freeze Leg A: create release branch ${BRANCH} @ ${NEW_VERSION} ===" + +git fetch origin "$UPSTREAM_BRANCH" + +main_version=$(read_elasticsearch_version_from_ref "origin/${UPSTREAM_BRANCH}") +if [[ -z "$main_version" ]]; then + echo "ERROR: could not read elasticsearchVersion from origin/${UPSTREAM_BRANCH}" >&2 + exit 1 +fi + +release_branch_exists=false +release_branch_version="" +if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + release_branch_exists=true + git fetch origin "$BRANCH" + release_branch_version=$(read_elasticsearch_version_from_ref "origin/${BRANCH}") +fi + +if ! "$PYTHON" "$VALIDATION_PY" validate-minor-freeze \ + --main-version "$main_version" \ + --new "$NEW_VERSION" \ + --branch "$BRANCH" \ + $([[ "$release_branch_exists" == "true" ]] && echo --release-branch-exists) \ + ${release_branch_version:+--release-branch-version "$release_branch_version"} +then + exit 1 +fi + +if [[ "$release_branch_exists" == "true" ]]; then + echo "Release branch origin/${BRANCH} already exists at ${NEW_VERSION} — nothing to do" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_created" "true" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "false" + exit 0 +fi + +if [[ "$main_version" != "$NEW_VERSION" ]]; then + echo "ERROR: origin/${UPSTREAM_BRANCH} is ${main_version}, expected ${NEW_VERSION} before branching" >&2 + exit 1 +fi + +if [ "$DRY_RUN" = "true" ]; then + echo " [DRY RUN] Would push origin ${UPSTREAM_BRANCH}:${BRANCH}" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_created" "true" + version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "true" + exit 0 +fi + +configure_git +git push origin "${UPSTREAM_BRANCH}:refs/heads/${BRANCH}" +echo " Created origin/${BRANCH} from origin/${UPSTREAM_BRANCH}" + +git fetch origin "$BRANCH" +branch_version=$(read_elasticsearch_version_from_ref "origin/${BRANCH}") +if [[ "$branch_version" != "$NEW_VERSION" ]]; then + echo "ERROR: origin/${BRANCH} version is ${branch_version}, expected ${NEW_VERSION}" >&2 + exit 1 +fi + +version_bump_set_buildkite_meta "ml_cpp_minor_branch_created" "true" +version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "true" +echo "OK: release branch ${BRANCH} is at ${NEW_VERSION}" diff --git a/dev-tools/unittest/test_job_version_bump_pipeline.py b/dev-tools/unittest/test_job_version_bump_pipeline.py index b77d59159..e278812f3 100644 --- a/dev-tools/unittest/test_job_version_bump_pipeline.py +++ b/dev-tools/unittest/test_job_version_bump_pipeline.py @@ -21,6 +21,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] _PIPELINE_PHASE1 = _REPO_ROOT / ".buildkite" / "job-version-bump.json.py" _PIPELINE_PHASE2 = _REPO_ROOT / ".buildkite" / "job-version-bump-phase2.json.py" +_PIPELINE_PHASE2_MINOR = _REPO_ROOT / ".buildkite" / "job-version-bump-phase2-minor.json.py" def _run_phase1(extra_env: dict[str, str] | None = None) -> dict: @@ -37,6 +38,20 @@ def _run_phase1(extra_env: dict[str, str] | None = None) -> dict: return json.loads(out) +def _run_phase2_minor(extra_env: dict[str, str] | None = None) -> dict: + env = os.environ.copy() + env.pop("VERSION_BUMP_MERGE_AUTO", None) + if extra_env: + env.update(extra_env) + out = subprocess.check_output( + [sys.executable, str(_PIPELINE_PHASE2_MINOR)], + cwd=str(_REPO_ROOT), + env=env, + text=True, + ) + return json.loads(out) + + def _run_phase2(extra_env: dict[str, str] | None = None) -> dict: env = os.environ.copy() env.pop("VERSION_BUMP_MERGE_AUTO", None) @@ -104,7 +119,7 @@ def test_phase2_dra_uses_wait_script_not_meta_in_if() -> None: dra = _step_by_key(pipeline, "fetch-dra-artifacts") assert "if" not in dra assert "plugins" not in dra - assert dra["command"] == ["python3", "dev-tools/wait_version_bump_dra.py"] + assert dra["command"] == ["python3 dev-tools/wait_version_bump_dra.py"] def test_phase2_order_bump_then_slack_then_dra() -> None: @@ -180,3 +195,27 @@ def test_create_pr_script_requires_body() -> None: ) assert proc.returncode != 0 assert "--body" in proc.stderr + + +def test_phase2_minor_has_parallel_freeze_group() -> None: + pipeline = _run_phase2_minor() + group = pipeline["steps"][0] + assert group["key"] == "minor-freeze" + keys = [s["key"] for s in group["steps"]] + assert keys == ["create-minor-branch", "bump-main-minor-freeze"] + + +def test_phase2_minor_order_group_then_slack_then_dra() -> None: + pipeline = _run_phase2_minor() + assert _step_by_key(pipeline, "queue-slack-notify")["depends_on"] == "minor-freeze" + assert ( + _step_by_key(pipeline, "fetch-dra-artifacts")["depends_on"] + == "queue-slack-notify" + ) + + +def test_phase2_minor_dra_command_is_single_shell_line() -> None: + """Buildkite runs each command array element as a separate shell line.""" + pipeline = _run_phase2_minor() + dra = _step_by_key(pipeline, "fetch-dra-artifacts") + assert dra["command"] == ["python3 dev-tools/wait_version_bump_dra.py"] diff --git a/dev-tools/unittest/test_version_bump_upload_phase2.py b/dev-tools/unittest/test_version_bump_upload_phase2.py new file mode 100644 index 000000000..a9b5fb456 --- /dev/null +++ b/dev-tools/unittest/test_version_bump_upload_phase2.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Tests for dev-tools/version_bump_upload_phase2.sh.""" + +from __future__ import annotations + +import json +import os +import subprocess +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_UPLOAD_SCRIPT = _REPO_ROOT / "dev-tools" / "version_bump_upload_phase2.sh" + + +@pytest.fixture +def fake_buildkite_agent(tmp_path: Path) -> tuple[Path, Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + count_file = tmp_path / "upload_count" + count_file.write_text("0") + agent = bin_dir / "buildkite-agent" + agent.write_text( + textwrap.dedent( + f"""\ + #!/bin/bash + set -euo pipefail + if [[ "$1" == "meta-data" && "$2" == "get" ]]; then + echo "false" + exit 0 + fi + if [[ "$1" == "pipeline" && "$2" == "upload" ]]; then + n=$(cat "{count_file}") + echo $((n + 1)) > "{count_file}" + cat > "{tmp_path}/upload-${{n}}.json" + exit 0 + fi + echo "unexpected: $*" >&2 + exit 1 + """ + ) + ) + agent.chmod(0o755) + return bin_dir, count_file + + +def test_minor_workflow_uploads_phase2_once(fake_buildkite_agent: tuple[Path, Path]) -> None: + """WORKFLOW=minor must not also upload the patch phase-2 pipeline.""" + bin_dir, count_file = fake_buildkite_agent + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" + env["WORKFLOW"] = "minor" + env.pop("DRY_RUN", None) + + proc = subprocess.run( + ["/bin/bash", str(_UPLOAD_SCRIPT)], + cwd=str(_REPO_ROOT), + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == 0, proc.stderr + proc.stdout + assert count_file.read_text().strip() == "1" + + pipeline = json.loads((count_file.parent / "upload-0.json").read_text()) + assert pipeline["steps"][0]["key"] == "minor-freeze" + + +def test_patch_workflow_uploads_patch_phase2_once(fake_buildkite_agent: tuple[Path, Path]) -> None: + bin_dir, count_file = fake_buildkite_agent + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" + env["WORKFLOW"] = "patch" + env.pop("DRY_RUN", None) + + proc = subprocess.run( + ["/bin/bash", str(_UPLOAD_SCRIPT)], + cwd=str(_REPO_ROOT), + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == 0, proc.stderr + proc.stdout + assert count_file.read_text().strip() == "1" + + pipeline = json.loads((count_file.parent / "upload-0.json").read_text()) + assert pipeline["steps"][0]["key"] == "bump-version" diff --git a/dev-tools/unittest/test_version_bump_validation.py b/dev-tools/unittest/test_version_bump_validation.py index b5c8deee9..4c711a676 100644 --- a/dev-tools/unittest/test_version_bump_validation.py +++ b/dev-tools/unittest/test_version_bump_validation.py @@ -80,6 +80,25 @@ def test_patch_ok_consecutive() -> None: ) +def test_patch_ok_with_sandbox_branch_name() -> None: + vbu.validate_version_bump_params( + current_version="9.5.0", + new_version="9.5.1", + branch="testing-9.5", + ) + + +def test_release_branch_identity_strips_testing_prefix() -> None: + assert vbu.release_branch_identity("9.5") == "9.5" + assert vbu.release_branch_identity("testing-9.5") == "9.5" + assert vbu.is_sandbox_release_branch("testing-9.5") + assert not vbu.is_sandbox_release_branch("9.5") + + +def test_parse_release_branch_accepts_sandbox_prefix() -> None: + assert vbu.parse_release_branch("testing-9.5") == (9, 5) + + def test_patch_ok_noop_same_version() -> None: vbu.validate_version_bump_params( current_version="9.5.1", @@ -175,14 +194,66 @@ def test_shell_skip_validation_env() -> None: assert out.returncode == 0, out.stderr + out.stdout +def test_derive_main_new_version() -> None: + assert vbu.derive_main_new_version("9.5.0") == "9.6.0" + + +def test_minor_freeze_ok() -> None: + main_new = vbu.validate_minor_freeze_params( + main_version="9.5.0", + new_version="9.5.0", + branch="9.5", + release_branch_exists=False, + release_branch_version=None, + ) + assert main_new == "9.6.0" + + +def test_minor_freeze_ok_sandbox_branch_name() -> None: + main_new = vbu.validate_minor_freeze_params( + main_version="9.5.0", + new_version="9.5.0", + branch="testing-9.5", + release_branch_exists=False, + release_branch_version=None, + ) + assert main_new == "9.6.0" + + +def test_minor_freeze_rejects_main_not_at_new_version() -> None: + with pytest.raises(ValueError, match="main elasticsearchVersion"): + vbu.validate_minor_freeze_params( + main_version="9.4.0", + new_version="9.5.0", + branch="9.5", + release_branch_exists=False, + release_branch_version=None, + ) + + +def test_main_minor_bump_ok() -> None: + vbu.validate_main_minor_bump( + current_version="9.5.0", + main_new_version="9.6.0", + release_branch_version="9.5.0", + ) + + +def test_main_minor_bump_noop() -> None: + vbu.validate_main_minor_bump( + current_version="9.6.0", + main_new_version="9.6.0", + release_branch_version="9.5.0", + ) + + @pytest.mark.skipif( not _VALIDATOR_SCRIPT.is_file(), reason="validate_version_bump_params.sh missing", ) -def test_shell_rejects_non_patch_workflow() -> None: - """Upstream may send WORKFLOW=minor; fail before git fetch.""" +def test_shell_rejects_unknown_workflow() -> None: env = os.environ.copy() - env["WORKFLOW"] = "minor" + env["WORKFLOW"] = "feature-freeze" env["NEW_VERSION"] = "9.5.1" env["BRANCH"] = "9.5" env.pop("SKIP_VERSION_VALIDATION", None) diff --git a/dev-tools/unittest/test_wait_version_bump_dra.py b/dev-tools/unittest/test_wait_version_bump_dra.py index c463620c4..63b1311ac 100644 --- a/dev-tools/unittest/test_wait_version_bump_dra.py +++ b/dev-tools/unittest/test_wait_version_bump_dra.py @@ -42,17 +42,25 @@ def advance_sleep(_seconds: float) -> None: nonlocal t t += float(_seconds) + 1.0 + def meta_side_effect(key: str) -> str | None: + if key == "ml_cpp_version_bump_noop": + return None + if key == "ml_cpp_version_bump_changed": + return "true" + return None + with ( patch.dict( "os.environ", { "BRANCH": "9.5", "NEW_VERSION": "9.5.1", + "WORKFLOW": "patch", "BUILDKITE": "false", }, clear=False, ), - patch.object(mod, "_meta_get", return_value="true"), + patch.object(mod, "_meta_get", side_effect=meta_side_effect), patch.object(mod, "_fetch_version", return_value=None), patch.object(mod.time, "monotonic", side_effect=fake_monotonic), patch.object(mod.time, "sleep", side_effect=advance_sleep), @@ -63,4 +71,23 @@ def advance_sleep(_seconds: float) -> None: assert mod.main() == 1 out = capsys.readouterr().out - assert "staging=None snapshot=None (still waiting)" in out + assert "still waiting: staging=None, snapshot=None" in out + + +def test_main_skips_dra_wait_for_sandbox_branch(capsys) -> None: + mod = _load_wait_module() + with patch.dict( + "os.environ", + { + "BRANCH": "testing-9.5", + "NEW_VERSION": "9.5.0", + "WORKFLOW": "minor", + "BUILDKITE": "false", + }, + clear=False, + ): + assert mod.main() == 0 + + err = capsys.readouterr().err + assert "Sandbox release branch" in err + assert "testing-9.5" in err diff --git a/dev-tools/update_backportrc.py b/dev-tools/update_backportrc.py new file mode 100755 index 000000000..382d2145c --- /dev/null +++ b/dev-tools/update_backportrc.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +"""Update .backportrc.json for minor release feature freeze.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def update_backportrc_for_minor_freeze( + data: dict[str, Any], + *, + new_release_branch: str, + main_new_version: str, +) -> bool: + """Apply minor-freeze updates in place. Returns True if anything changed.""" + changed = False + + choices: list[str] = list(data.get("targetBranchChoices", [])) + if new_release_branch not in choices: + if "main" in choices: + insert_at = choices.index("main") + 1 + else: + insert_at = 0 + choices.insert(insert_at, new_release_branch) + data["targetBranchChoices"] = choices + changed = True + + mapping: dict[str, str] = dict(data.get("branchLabelMapping", {})) + new_main_key = f"^v{main_new_version}$" + old_main_keys = [k for k, v in mapping.items() if v == "main" and k != new_main_key] + for key in old_main_keys: + del mapping[key] + changed = True + if mapping.get(new_main_key) != "main": + mapping[new_main_key] = "main" + changed = True + data["branchLabelMapping"] = mapping + + return changed + + +def _cmd_update(args: argparse.Namespace) -> int: + path = Path(args.path) + if not path.is_file(): + print(f"ERROR: {path} not found", file=sys.stderr) + return 1 + + with path.open(encoding="utf-8") as handle: + data = json.load(handle) + + changed = update_backportrc_for_minor_freeze( + data, + new_release_branch=args.new_release_branch, + main_new_version=args.main_new_version, + ) + if not changed: + print(f"OK: {path} already configured for branch {args.new_release_branch} and main {args.main_new_version}") + return 0 + + with path.open("w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2) + handle.write("\n") + + print( + f"Updated {path}: added branch {args.new_release_branch}, " + f"main label mapping v{args.main_new_version}" + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Update .backportrc.json for minor freeze") + parser.add_argument("--path", default=".backportrc.json", help="Path to backportrc file") + parser.add_argument("--new-release-branch", required=True, help="New release branch (MAJOR.MINOR)") + parser.add_argument("--main-new-version", required=True, help="New version on main (MAJOR.MINOR.PATCH)") + args = parser.parse_args() + return _cmd_update(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev-tools/validate_version_bump_params.sh b/dev-tools/validate_version_bump_params.sh index 2742973c7..a5b9e9113 100755 --- a/dev-tools/validate_version_bump_params.sh +++ b/dev-tools/validate_version_bump_params.sh @@ -16,8 +16,7 @@ # Environment: # NEW_VERSION — required target stack version (MAJOR.MINOR.PATCH), unless skipped # BRANCH — required release branch (e.g. 9.5), unless skipped -# WORKFLOW — optional; defaults to patch. If set by upstream automation, must be -# exactly "patch" (this pipeline does not support minor bumps). +# WORKFLOW — optional; defaults to patch. Supported: patch, minor (feature freeze). # SKIP_VERSION_VALIDATION — set to "true" to skip (emergency override only) # PYTHON — interpreter (default: python3) # @@ -26,19 +25,10 @@ set -euo pipefail -version_bump_set_noop_meta() { - local noop="$1" - if [[ "${BUILDKITE:-}" != "true" ]]; then - return 0 - fi - if ! command -v buildkite-agent >/dev/null 2>&1; then - echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ml_cpp_version_bump_noop=${noop}" >&2 - return 0 - fi - buildkite-agent meta-data set "ml_cpp_version_bump_noop" "$noop" -} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=version_bump_lib.sh +source "${SCRIPT_DIR}/version_bump_lib.sh" + PYTHON="${PYTHON:-python3}" VALIDATION_PY="${SCRIPT_DIR}/version_bump_validation.py" @@ -53,19 +43,94 @@ fi : "${NEW_VERSION:?NEW_VERSION must be set}" : "${BRANCH:?BRANCH must be set}" -version_bump_trim_value() { - local s=$1 - s="${s//$'\r'/}" - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} NEW_VERSION="$(version_bump_trim_value "${NEW_VERSION}")" BRANCH="$(version_bump_trim_value "${BRANCH}")" WORKFLOW="${WORKFLOW:-patch}" +WORKFLOW="$(version_bump_trim_value "${WORKFLOW}")" + +if [[ "$WORKFLOW" == "minor" ]]; then + echo "=== Version bump validation (minor feature freeze) ===" + echo "WORKFLOW: ${WORKFLOW}" + echo "NEW_VERSION: ${NEW_VERSION} (expected on release branch ${BRANCH})" + echo "BRANCH: ${BRANCH} (release branch to create)" + if [[ "$BRANCH" == testing-* ]]; then + echo " (sandbox: version rules use identity ${BRANCH#testing-})" + fi + + echo "Fetching origin/main and checking origin/${BRANCH}..." + git fetch origin main + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + git fetch origin "$BRANCH" + fi + + main_version=$( + git show origin/main:gradle.properties | grep '^elasticsearchVersion=' | head -1 | cut -d= -f2 | tr -d '[:space:]' || true + ) + if [[ -z "$main_version" ]]; then + echo "ERROR: could not read elasticsearchVersion from origin/main gradle.properties" >&2 + exit 1 + fi + echo "Current version on origin/main: ${main_version}" + + release_branch_exists=false + release_branch_version="" + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + release_branch_exists=true + release_branch_version=$( + git show "origin/${BRANCH}:gradle.properties" | grep '^elasticsearchVersion=' | head -1 | cut -d= -f2 | tr -d '[:space:]' || true + ) + echo "Release branch origin/${BRANCH} exists at version: ${release_branch_version:-unknown}" + else + echo "Release branch origin/${BRANCH} does not exist yet" + fi + + minor_validate_args=( + "$PYTHON" "$VALIDATION_PY" validate-minor-freeze + --main-version "$main_version" + --new "$NEW_VERSION" + --branch "$BRANCH" + ) + if [[ "$release_branch_exists" == "true" ]]; then + minor_validate_args+=(--release-branch-exists --release-branch-version "$release_branch_version") + fi + if ! "${minor_validate_args[@]}"; then + exit 1 + fi + + MAIN_NEW_VERSION=$("$PYTHON" "$VALIDATION_PY" derive-main-new-version --new "$NEW_VERSION") + version_bump_set_buildkite_meta "ml_cpp_version_bump_main_new_version" "$MAIN_NEW_VERSION" + echo "Derived MAIN_NEW_VERSION for main bump: ${MAIN_NEW_VERSION}" + + branch_needed=true + if [[ "$release_branch_exists" == "true" && "$release_branch_version" == "$NEW_VERSION" ]]; then + branch_needed=false + fi + version_bump_set_buildkite_meta "ml_cpp_minor_branch_needed" "$([[ "$branch_needed" == "true" ]] && echo true || echo false)" + + main_bump_needed=true + main_trim=$(echo "$main_version" | tr -d '[:space:]') + main_new_trim=$(echo "$MAIN_NEW_VERSION" | tr -d '[:space:]') + if [[ "$main_trim" == "$main_new_trim" ]]; then + main_bump_needed=false + fi + if [[ "$BRANCH" == testing-* ]]; then + main_bump_needed=false + echo "Sandbox branch ${BRANCH} — main bump will be skipped" + fi + version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "$([[ "$main_bump_needed" == "true" ]] && echo true || echo false)" + + if [[ "$branch_needed" == "false" && "$main_bump_needed" == "false" ]]; then + version_bump_set_noop_meta true + echo "OK: release branch and main bump already complete — follow-up steps will no-op." + else + version_bump_set_noop_meta false + fi + exit 0 +fi + if [[ "$WORKFLOW" != "patch" ]]; then - echo "ERROR: WORKFLOW must be \"patch\" for this pipeline, got \"${WORKFLOW}\"" >&2 + echo "ERROR: WORKFLOW must be \"patch\" or \"minor\", got \"${WORKFLOW}\"" >&2 exit 1 fi @@ -74,9 +139,7 @@ echo "WORKFLOW: ${WORKFLOW}" echo "NEW_VERSION: ${NEW_VERSION}" echo "BRANCH: ${BRANCH}" -# Patch-only pipeline (no WORKFLOW=minor): consecutive patch on this release -# branch. Current version is read from origin/${BRANCH} by design — there is no -# minor-line bump mode in dev-tools/version_bump_validation.py or this pipeline. +# Patch workflow: consecutive patch increment on the release branch named BRANCH. echo "Fetching origin/${BRANCH}..." git fetch origin "$BRANCH" diff --git a/dev-tools/version_bump_lib.sh b/dev-tools/version_bump_lib.sh new file mode 100755 index 000000000..32ec1c93b --- /dev/null +++ b/dev-tools/version_bump_lib.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Shared helpers for ml-cpp release-eng version bump scripts. + +set -euo pipefail + +version_bump_trim_value() { + local s=$1 + s="${s//$'\r'/}" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +configure_git() { + git config user.name elasticsearchmachine + git config user.email 'infra-root+elasticsearchmachine@elastic.co' +} + +sed_inplace() { + local script=$1 + local target=$2 + local tmp + tmp=$(mktemp "${target}.sedtmp.XXXXXX") + if ! sed "${script}" "$target" >"$tmp"; then + rm -f "$tmp" + return 1 + fi + mv "$tmp" "$target" +} + +github_repo_slug() { + local url + url=$(git remote get-url origin 2>/dev/null || true) + if [[ "$url" =~ github\.com[:/]([^/]+)/([^/.]+)(\.git)?$ ]]; then + echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" + return 0 + fi + echo "ERROR: could not parse owner/repo from git remote url: ${url:-empty}" >&2 + return 1 +} + +version_bump_set_buildkite_meta() { + local key="$1" + local value="$2" + if [[ "${BUILDKITE:-}" != "true" ]]; then + return 0 + fi + if ! command -v buildkite-agent >/dev/null 2>&1; then + echo "WARNING: BUILDKITE=true but buildkite-agent not in PATH; skipping meta-data ${key}=${value}" >&2 + return 0 + fi + buildkite-agent meta-data set "$key" "$value" +} + +version_bump_set_buildkite_meta_changed() { + version_bump_set_buildkite_meta "ml_cpp_version_bump_changed" "$1" +} + +version_bump_set_noop_meta() { + local noop="$1" + version_bump_set_buildkite_meta "ml_cpp_version_bump_noop" "$noop" +} + +version_bump_set_pr_url_meta() { + local url="${1:-}" + if [[ -z "${url}" ]]; then + return 0 + fi + version_bump_set_buildkite_meta "ml_cpp_version_bump_pr_url" "$url" +} + +read_elasticsearch_version_from_file() { + local file=$1 + grep '^elasticsearchVersion=' "$file" | head -1 | cut -d= -f2 | tr -d '[:space:]' || true +} + +read_elasticsearch_version_from_ref() { + local ref=$1 + git show "${ref}:gradle.properties" | grep '^elasticsearchVersion=' | head -1 | cut -d= -f2 | tr -d '[:space:]' || true +} diff --git a/dev-tools/version_bump_upload_phase2.sh b/dev-tools/version_bump_upload_phase2.sh index 82be7561c..7996db367 100755 --- a/dev-tools/version_bump_upload_phase2.sh +++ b/dev-tools/version_bump_upload_phase2.sh @@ -42,4 +42,12 @@ if [[ "${noop}" == "true" ]]; then exit 0 fi -exec python3 .buildkite/job-version-bump-phase2.json.py | buildkite-agent pipeline upload +WORKFLOW="${WORKFLOW:-patch}" +# Do not use `exec cmd | buildkite-agent pipeline upload`: in bash, exec applies inside +# the pipeline subshell only, so the script would continue and upload a second phase-2 +# pipeline (duplicate step keys such as queue-slack-notify). +if [[ "${WORKFLOW}" == "minor" ]]; then + python3 .buildkite/job-version-bump-phase2-minor.json.py | buildkite-agent pipeline upload +else + python3 .buildkite/job-version-bump-phase2.json.py | buildkite-agent pipeline upload +fi diff --git a/dev-tools/version_bump_validation.py b/dev-tools/version_bump_validation.py index 42cbda0e9..15727b3de 100644 --- a/dev-tools/version_bump_validation.py +++ b/dev-tools/version_bump_validation.py @@ -8,7 +8,15 @@ # compliance with the Elastic License 2.0 and the foregoing additional # limitation. # -"""Rules for ml-cpp patch release version bump parameters (Buildkite / release-eng). +"""Rules for ml-cpp release version bump parameters (Buildkite / release-eng). + +Patch and minor (feature freeze) workflows share parameter names from release-eng: +NEW_VERSION, BRANCH, WORKFLOW. For WORKFLOW=minor, NEW_VERSION is the version +expected on the new release branch (e.g. 9.5.0 on branch 9.5); main is bumped to +derive_main_new_version(NEW_VERSION) (e.g. 9.6.0). + +BRANCH may be MAJOR.MINOR or a sandbox ref ``testing-MAJOR.MINOR`` (e.g. ``testing-9.5``). +Version rules strip the ``testing-`` prefix; git operations use the full ref name. Used by dev-tools/validate_version_bump_params.sh and unit-tested under dev-tools/unittest/. @@ -33,6 +41,19 @@ SEMVER_RE = re.compile(r"^([0-9]+)\.([0-9]+)\.([0-9]+)$") BRANCH_RE = re.compile(r"^([0-9]+)\.([0-9]+)$") +SANDBOX_BRANCH_PREFIX = "testing-" + + +def is_sandbox_release_branch(branch: str) -> bool: + """True when BRANCH is a manual-test ref (testing-MAJOR.MINOR), not a production line.""" + return branch.startswith(SANDBOX_BRANCH_PREFIX) + + +def release_branch_identity(branch: str) -> str: + """Return MAJOR.MINOR identity for version rules (strip leading testing- prefix).""" + if is_sandbox_release_branch(branch): + return branch[len(SANDBOX_BRANCH_PREFIX) :] + return branch def _reject_outer_whitespace(label: str, value: str) -> None: @@ -51,7 +72,8 @@ def parse_semver(version: str) -> Optional[Tuple[int, int, int]]: def parse_release_branch(branch: str) -> Optional[Tuple[int, int]]: - m = BRANCH_RE.match(branch) + identity = release_branch_identity(branch) + m = BRANCH_RE.match(identity) if not m: return None return (int(m.group(1)), int(m.group(2))) @@ -81,7 +103,8 @@ def validate_version_bump_params( br = parse_release_branch(branch) if br is None: raise ValueError( - f"BRANCH must be MAJOR.MINOR (e.g. 9.5), got {branch!r}" + f"BRANCH must be MAJOR.MINOR (e.g. 9.5) or " + f"{SANDBOX_BRANCH_PREFIX}MAJOR.MINOR (e.g. testing-9.5), got {branch!r}" ) br_major, br_minor = br if br_major != new_major or br_minor != new_minor: @@ -115,6 +138,119 @@ def validate_version_bump_params( ) +def derive_main_new_version(release_branch_version: str) -> str: + """Return the main-branch version after minor feature freeze (minor + 1, patch 0).""" + parsed = parse_semver(release_branch_version) + if parsed is None: + raise ValueError( + f"release branch version must be MAJOR.MINOR.PATCH, got {release_branch_version!r}" + ) + major, minor, patch = parsed + if patch != 0: + raise ValueError( + "minor freeze expects NEW_VERSION patch 0 " + f"(got {release_branch_version!r})" + ) + return f"{major}.{minor + 1}.0" + + +def validate_minor_freeze_params( + *, + main_version: str, + new_version: str, + branch: str, + release_branch_exists: bool, + release_branch_version: str | None, +) -> str: + """Validate minor freeze inputs. Returns MAIN_NEW_VERSION (derived). + + NEW_VERSION is the version expected on the new release branch (BRANCH). + main must currently be at NEW_VERSION before the freeze bump. + """ + _reject_outer_whitespace("NEW_VERSION", new_version) + _reject_outer_whitespace("BRANCH", branch) + _reject_outer_whitespace("main_version", main_version) + + new_t = parse_semver(new_version) + if new_t is None: + raise ValueError( + f"NEW_VERSION must be MAJOR.MINOR.PATCH (digits only), got {new_version!r}" + ) + new_major, new_minor, new_patch = new_t + if new_patch != 0: + raise ValueError( + f"minor freeze NEW_VERSION must be X.Y.0 (patch 0), got {new_version!r}" + ) + + br = parse_release_branch(branch) + if br is None: + raise ValueError( + f"BRANCH must be MAJOR.MINOR (e.g. 9.5) or " + f"{SANDBOX_BRANCH_PREFIX}MAJOR.MINOR (e.g. testing-9.5), got {branch!r}" + ) + br_major, br_minor = br + if br_major != new_major or br_minor != new_minor: + raise ValueError( + f"BRANCH {branch!r} must match MAJOR.MINOR of NEW_VERSION " + f"({new_major}.{new_minor}), got NEW_VERSION {new_version!r}" + ) + + main_t = parse_semver(main_version) + if main_t is None: + raise ValueError( + "elasticsearchVersion on main must be MAJOR.MINOR.PATCH, " + f"got {main_version!r}" + ) + if main_version != new_version: + raise ValueError( + "minor freeze requires main elasticsearchVersion to equal NEW_VERSION " + f"before branching (main={main_version!r}, NEW_VERSION={new_version!r})" + ) + + main_new_version = derive_main_new_version(new_version) + + if release_branch_exists: + if release_branch_version is None: + raise ValueError( + f"release branch {branch!r} exists but version could not be read" + ) + if release_branch_version != new_version: + raise ValueError( + f"release branch {branch!r} exists with version {release_branch_version!r}, " + f"expected {new_version!r}" + ) + + return main_new_version + + +def validate_main_minor_bump( + *, + current_version: str, + main_new_version: str, + release_branch_version: str, +) -> None: + """Validate bumping main from release-branch version to MAIN_NEW_VERSION.""" + _reject_outer_whitespace("current_version", current_version) + _reject_outer_whitespace("main_new_version", main_new_version) + _reject_outer_whitespace("release_branch_version", release_branch_version) + + if current_version == main_new_version: + return + + if current_version != release_branch_version: + raise ValueError( + "main bump expects current main version to equal NEW_VERSION " + f"({release_branch_version!r}), got {current_version!r}" + ) + + expected = derive_main_new_version(release_branch_version) + if main_new_version != expected: + raise ValueError( + f"MAIN_NEW_VERSION must be {expected!r} for NEW_VERSION " + f"{release_branch_version!r}, got {main_new_version!r}" + ) + + def _cmd_validate(args: argparse.Namespace) -> int: try: validate_version_bump_params( @@ -153,7 +289,7 @@ def main() -> int: ) p_val.add_argument("--current", required=True, help="elasticsearchVersion on branch") p_val.add_argument("--new", required=True, dest="new", help="NEW_VERSION") - p_val.add_argument("--branch", required=True, help="BRANCH (MAJOR.MINOR)") + p_val.add_argument("--branch", required=True, help="BRANCH (MAJOR.MINOR or testing-MAJOR.MINOR)") p_val.set_defaults(func=_cmd_validate) p_rep = sub.add_parser( @@ -165,6 +301,93 @@ def main() -> int: p_rep.add_argument("--branch", required=True) p_rep.set_defaults(func=_cmd_validate_and_report) + p_minor = sub.add_parser( + "validate-minor-freeze", + help="check main/new/branch for WORKFLOW=minor", + ) + p_minor.add_argument("--main-version", required=True) + p_minor.add_argument("--new", required=True, dest="new") + p_minor.add_argument("--branch", required=True) + p_minor.add_argument( + "--release-branch-exists", + action="store_true", + help="origin/BRANCH already exists", + ) + p_minor.add_argument( + "--release-branch-version", + default="", + help="elasticsearchVersion on origin/BRANCH when it exists", + ) + + def _cmd_validate_minor(args_ns: argparse.Namespace) -> int: + try: + rb_ver = args_ns.release_branch_version or None + main_new = validate_minor_freeze_params( + main_version=args_ns.main_version, + new_version=args_ns.new, + branch=args_ns.branch, + release_branch_exists=args_ns.release_branch_exists, + release_branch_version=rb_ver, + ) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + print(f"OK: minor freeze NEW_VERSION={args_ns.new} on branch {args_ns.branch}") + if is_sandbox_release_branch(args_ns.branch): + identity = release_branch_identity(args_ns.branch) + print( + f"OK: sandbox branch (version identity {identity!r}); " + "main bump and DRA wait are skipped in CI" + ) + print(f"OK: main bump target MAIN_NEW_VERSION={main_new}") + return 0 + + p_minor.set_defaults(func=_cmd_validate_minor) + + p_main = sub.add_parser( + "validate-main-minor-bump", + help="check main bump during minor freeze", + ) + p_main.add_argument("--current", required=True) + p_main.add_argument("--main-new-version", required=True) + p_main.add_argument("--release-branch-version", required=True) + + def _cmd_validate_main_bump(args_ns: argparse.Namespace) -> int: + try: + validate_main_minor_bump( + current_version=args_ns.current, + main_new_version=args_ns.main_new_version, + release_branch_version=args_ns.release_branch_version, + ) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + if args_ns.current == args_ns.main_new_version: + print(f"OK: main already at {args_ns.main_new_version} — bump step will no-op.") + else: + print( + f"OK: main minor bump {args_ns.current} → {args_ns.main_new_version}" + ) + return 0 + + p_main.set_defaults(func=_cmd_validate_main_bump) + + p_derive = sub.add_parser( + "derive-main-new-version", + help="print MAIN_NEW_VERSION for NEW_VERSION (minor freeze)", + ) + p_derive.add_argument("--new", required=True, dest="new") + + def _cmd_derive_main(args_ns: argparse.Namespace) -> int: + try: + print(derive_main_new_version(args_ns.new)) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + return 0 + + p_derive.set_defaults(func=_cmd_derive_main) + args = parser.parse_args() return args.func(args) diff --git a/dev-tools/wait_version_bump_dra.py b/dev-tools/wait_version_bump_dra.py index 50e40ce9b..ed968597d 100755 --- a/dev-tools/wait_version_bump_dra.py +++ b/dev-tools/wait_version_bump_dra.py @@ -11,8 +11,14 @@ """Poll DRA staging/snapshot JSON until versions match (replaces json-watcher plugin). Buildkite step conditionals cannot use build meta-data; this script reads -ml_cpp_version_bump_changed via ``buildkite-agent meta-data get`` and exits -immediately when no PR was opened. +ml_cpp_version_bump_noop / ml_cpp_version_bump_changed via ``buildkite-agent +meta-data get`` and exits immediately when the wait is not needed. + +Patch (WORKFLOW=patch): waits for staging + snapshot on BRANCH at NEW_VERSION. + +Minor (WORKFLOW=minor): waits for three artifact sets after feature freeze: + - snapshot on main at MAIN_NEW_VERSION-SNAPSHOT + - snapshot + staging on release branch BRANCH at NEW_VERSION """ from __future__ import annotations @@ -24,10 +30,16 @@ import time import urllib.error import urllib.request +from pathlib import Path + +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +import version_bump_validation as vbu POLL_SECONDS = 30 TIMEOUT_SECONDS = 240 * 60 -# Heartbeat in Buildkite logs every N poll iterations (even when fetches return None). PROGRESS_LOG_EVERY = 1 STAGING_TMPL = "https://artifacts-staging.elastic.co/ml-cpp/latest/{branch}.json" @@ -35,11 +47,7 @@ def _meta_get(key: str) -> str | None: - """Read Buildkite meta-data. Returns None when not on Buildkite or key is unset. - - On BUILDKITE=true, missing ``buildkite-agent`` or unexpected failures exit - non-zero so we do not silently skip the DRA wait. - """ + """Read Buildkite meta-data. Returns None when not on Buildkite or key is unset.""" if os.environ.get("BUILDKITE") != "true": return None try: @@ -100,48 +108,115 @@ def _fetch_version(url: str) -> str | None: return None +def _derive_main_new_version(release_version: str) -> str: + parts = release_version.split(".") + if len(parts) != 3: + raise ValueError(f"invalid semver: {release_version!r}") + major, minor, patch = int(parts[0]), int(parts[1]), int(parts[2]) + if patch != 0: + raise ValueError(f"expected patch 0 for minor freeze, got {release_version!r}") + return f"{major}.{minor + 1}.0" + + +def _wait_for_checks(checks: list[tuple[str, str, str]]) -> int: + """Poll until all (label, url, expected_version) match.""" + print(f"Waiting for DRA artifacts (timeout {TIMEOUT_SECONDS}s, poll {POLL_SECONDS}s)...") + for label, url, expected in checks: + print(f" {label}: {expected!r} <= {url}") + + deadline = time.monotonic() + TIMEOUT_SECONDS + iteration = 0 + while time.monotonic() < deadline: + iteration += 1 + pending = [] + for label, url, expected in checks: + got = _fetch_version(url) + if got != expected: + pending.append(f"{label}={got!r}") + if not pending: + print("OK: all DRA artifact versions matched.") + return 0 + if iteration % PROGRESS_LOG_EVERY == 0: + print(f" still waiting: {', '.join(pending)}") + time.sleep(POLL_SECONDS) + + print("ERROR: timed out waiting for DRA artifact versions.", file=sys.stderr) + return 1 + + +def _wait_patch(branch: str, new_version: str) -> int: + staging_url = STAGING_TMPL.format(branch=branch) + snapshot_url = SNAPSHOT_TMPL.format(branch=branch) + checks = [ + ("staging", staging_url, new_version), + ("snapshot", snapshot_url, f"{new_version}-SNAPSHOT"), + ] + return _wait_for_checks(checks) + + +def _wait_minor(branch: str, new_version: str, main_new_version: str) -> int: + main_snapshot_url = SNAPSHOT_TMPL.format(branch="main") + branch_staging_url = STAGING_TMPL.format(branch=branch) + branch_snapshot_url = SNAPSHOT_TMPL.format(branch=branch) + checks = [ + ("main snapshot", main_snapshot_url, f"{main_new_version}-SNAPSHOT"), + ("release snapshot", branch_snapshot_url, f"{new_version}-SNAPSHOT"), + ("release staging", branch_staging_url, new_version), + ] + return _wait_for_checks(checks) + + def main() -> int: if os.environ.get("DRY_RUN") == "true": print("DRY_RUN=true — skipping DRA wait.") return 0 - if _meta_get("ml_cpp_version_bump_changed") != "true": + if _meta_get("ml_cpp_version_bump_noop") == "true": print( - "ml_cpp_version_bump_changed is not true — no PR opened; skipping DRA wait.", + "ml_cpp_version_bump_noop is true — nothing to wait for; skipping DRA wait.", file=sys.stderr, ) return 0 + workflow = os.environ.get("WORKFLOW", "patch").strip().lower() branch = os.environ.get("BRANCH", "").strip() new_version = os.environ.get("NEW_VERSION", "").strip() if not branch or not new_version: print("ERROR: BRANCH and NEW_VERSION must be set.", file=sys.stderr) return 1 - staging_url = STAGING_TMPL.format(branch=branch) - snapshot_url = SNAPSHOT_TMPL.format(branch=branch) - want_staging = new_version - want_snapshot = f"{new_version}-SNAPSHOT" + if vbu.is_sandbox_release_branch(branch): + print( + f"Sandbox release branch {branch!r} — skipping DRA wait " + f"(no artifacts published for {vbu.SANDBOX_BRANCH_PREFIX}* refs).", + file=sys.stderr, + ) + return 0 - print(f"Waiting for DRA artifacts (timeout {TIMEOUT_SECONDS}s, poll {POLL_SECONDS}s)...") - print(f" staging: {want_staging!r} <= {staging_url}") - print(f" snapshot: {want_snapshot!r} <= {snapshot_url}") + if workflow == "minor": + main_new_version = _meta_get("ml_cpp_version_bump_main_new_version") + if not main_new_version: + main_new_version = os.environ.get("MAIN_NEW_VERSION", "").strip() + if not main_new_version: + try: + main_new_version = _derive_main_new_version(new_version) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + print( + f"Minor freeze DRA wait: release branch {branch} @ {new_version}, " + f"main @ {main_new_version}" + ) + return _wait_minor(branch, new_version, main_new_version) - deadline = time.monotonic() + TIMEOUT_SECONDS - iteration = 0 - while time.monotonic() < deadline: - iteration += 1 - st = _fetch_version(staging_url) - sn = _fetch_version(snapshot_url) - if st == want_staging and sn == want_snapshot: - print("OK: staging and snapshot versions matched.") - return 0 - if iteration % PROGRESS_LOG_EVERY == 0: - print(f" staging={st!r} snapshot={sn!r} (still waiting)") - time.sleep(POLL_SECONDS) + if _meta_get("ml_cpp_version_bump_changed") != "true": + print( + "ml_cpp_version_bump_changed is not true — no PR opened; skipping DRA wait.", + file=sys.stderr, + ) + return 0 - print("ERROR: timed out waiting for DRA artifact versions.", file=sys.stderr) - return 1 + return _wait_patch(branch, new_version) if __name__ == "__main__":