release: stage Jina embedding migration for 0.2.5851 #597
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: bench-regression | |
| on: | |
| pull_request: | |
| permissions: | |
| contents: read | |
| env: | |
| ZIG_VERSION: 0.17.0-dev.813+2153f8143 | |
| ZIG_SHA256: b0d46ffc4587b9e8dd0b524ee5bc4da1e67f28bba55e7c534cec64af2f2d7a74 | |
| LEGACY_ZIG_VERSION: 0.16.0 | |
| LEGACY_ZIG_SHA256: 70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 | |
| PAIRED_5829_BASE_SHA: dd36e9431925014ee2bed80346669a4afee7e42e | |
| PAIRED_5829_SHA: 24e89c70d4f9cdaf5542a78d83d1890a42b4a046 | |
| PAIRED_5846_BASE_SHA: 3db4242b9b39a857bdb4657d39bb623c76501fe9 | |
| PAIRED_5846_SHA: f5657e30f2e5bad759b5de20b591e2f67436e1b9 | |
| jobs: | |
| macos-resource-regression: | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install pinned Zig toolchain | |
| run: | | |
| set -euo pipefail | |
| case "$(uname -m)" in | |
| arm64) | |
| zig_arch=aarch64 | |
| zig_sha=36673d2513afa4a96c86780648ba504beedd7f0451389091cf9d53e38d5b4840 | |
| ;; | |
| x86_64) | |
| zig_arch=x86_64 | |
| zig_sha=3938c46ae4bca3c13f423b09503e3ef00bb4b7ef12b8bc1e5122ede366057a5b | |
| ;; | |
| *) | |
| echo "unsupported macOS architecture: $(uname -m)" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| archive="zig-${zig_arch}-macos-${ZIG_VERSION}.tar.xz" | |
| directory="zig-${zig_arch}-macos-${ZIG_VERSION}" | |
| curl --fail --show-error --location --retry 3 \ | |
| "https://ziglang.org/builds/${archive}" -o zig-macos.tar.xz || \ | |
| curl --fail --show-error --location --retry 3 \ | |
| "https://pkg.machengine.org/zig/${archive}" -o zig-macos.tar.xz | |
| echo "${zig_sha} zig-macos.tar.xz" | shasum -a 256 --check | |
| tar -xf zig-macos.tar.xz | |
| test "$("$PWD/${directory}/zig" version)" = "$ZIG_VERSION" | |
| echo "$PWD/${directory}" >> "$GITHUB_PATH" | |
| - name: Run macOS watcher unit and process gates | |
| run: | | |
| zig build test-index -Dtest-filter=issue-709 | |
| zig build -Doptimize=ReleaseFast | |
| python3 scripts/fd_regression_test.py --binary zig-out/bin/codedb | |
| bench: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Test paired benchmark comparator | |
| run: python3 -m unittest scripts/test_compare_bench_paired.py | |
| - name: Install pinned Zig toolchains | |
| run: | | |
| set -euo pipefail | |
| head_archive="zig-x86_64-linux-${ZIG_VERSION}.tar.xz" | |
| head_dir="zig-x86_64-linux-${ZIG_VERSION}" | |
| # Old dev builds are rotated off ziglang.org; fall back to the | |
| # hexops community mirror (byte-identical official tarballs). | |
| # The pinned sha256 below is verified regardless of source. | |
| curl --fail --show-error --location --retry 3 \ | |
| "https://ziglang.org/builds/${head_archive}" -o zig-head.tar.xz || \ | |
| curl --fail --show-error --location --retry 3 \ | |
| "https://pkg.machengine.org/zig/${head_archive}" -o zig-head.tar.xz | |
| echo "${ZIG_SHA256} zig-head.tar.xz" | sha256sum --check | |
| tar -xf zig-head.tar.xz | |
| test "$("$PWD/${head_dir}/zig" version)" = "$ZIG_VERSION" | |
| legacy_archive="zig-x86_64-linux-${LEGACY_ZIG_VERSION}.tar.xz" | |
| legacy_dir="zig-x86_64-linux-${LEGACY_ZIG_VERSION}" | |
| curl --fail --show-error --location --retry 3 \ | |
| "https://ziglang.org/download/${LEGACY_ZIG_VERSION}/${legacy_archive}" -o zig-legacy.tar.xz | |
| echo "${LEGACY_ZIG_SHA256} zig-legacy.tar.xz" | sha256sum --check | |
| tar -xf zig-legacy.tar.xz | |
| test "$("$PWD/${legacy_dir}/zig" version)" = "$LEGACY_ZIG_VERSION" | |
| echo "$PWD/${head_dir}" >> "$GITHUB_PATH" | |
| - name: Benchmark head | |
| run: | | |
| python3 scripts/test_run_bench_json.py | |
| python3 scripts/run-bench-json.py bench-head.json | |
| - name: Benchmark base with its declared compiler | |
| id: base_bench | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| run: | | |
| set -euo pipefail | |
| git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null || git fetch origin "$BASE_SHA" --depth=1 | |
| git worktree add ../codedb-base "$BASE_SHA" | |
| test "$(git -C ../codedb-base rev-parse HEAD)" = "$BASE_SHA" | |
| cd ../codedb-base | |
| base_zig_version=$(sed -n 's/.*\.minimum_zig_version = "\([^"]*\)".*/\1/p' build.zig.zon) | |
| case "$base_zig_version" in | |
| "$ZIG_VERSION") | |
| base_zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${ZIG_VERSION}" | |
| ;; | |
| "$LEGACY_ZIG_VERSION") | |
| base_zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${LEGACY_ZIG_VERSION}" | |
| ;; | |
| *) | |
| echo "unsupported base Zig version: ${base_zig_version:-missing}" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| test "$("$base_zig_dir/zig" version)" = "$base_zig_version" | |
| echo "zig_version=$base_zig_version" >> "$GITHUB_OUTPUT" | |
| echo "zig_dir=$base_zig_dir" >> "$GITHUB_OUTPUT" | |
| PATH="$base_zig_dir:$PATH" \ | |
| python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json" | |
| - name: Paired counterbalanced benchmark with parity gate | |
| env: | |
| EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| BASE_ZIG_VERSION: ${{ steps.base_bench.outputs.zig_version }} | |
| BASE_ZIG_DIR: ${{ steps.base_bench.outputs.zig_dir }} | |
| PAIRS: 5 | |
| run: | | |
| set -euo pipefail | |
| if [[ "$BASE_ZIG_VERSION" != "$ZIG_VERSION" ]]; then | |
| echo "Paired parity requires a pinned same-compiler baseline harness; refusing to skip the gate." >&2 | |
| exit 1 | |
| fi | |
| paired_base="../codedb-base" | |
| if [[ "$EVENT_BASE_SHA" == "$PAIRED_5846_BASE_SHA" ]]; then | |
| git cat-file -e "$PAIRED_5846_SHA^{commit}" 2>/dev/null || { | |
| git fetch origin bench/v0.2.5846-content-parity --depth=1 | |
| test "$(git rev-parse FETCH_HEAD)" = "$PAIRED_5846_SHA" | |
| } | |
| git merge-base --is-ancestor "$EVENT_BASE_SHA" "$PAIRED_5846_SHA" | |
| test "$(git diff --name-only "$EVENT_BASE_SHA..$PAIRED_5846_SHA")" = 'src/mcp.zig' | |
| git worktree add ../codedb-paired-base "$PAIRED_5846_SHA" | |
| paired_base="../codedb-paired-base" | |
| elif ! grep -q corpus_hash "$paired_base/src/bench.zig"; then | |
| if [[ "$EVENT_BASE_SHA" != "$PAIRED_5829_BASE_SHA" ]]; then | |
| echo "No pinned parity harness exists for base $EVENT_BASE_SHA" >&2 | |
| exit 1 | |
| fi | |
| git cat-file -e "$PAIRED_5829_SHA^{commit}" 2>/dev/null || { | |
| git fetch origin bench/v0.2.5829-paired-baseline --depth=1 | |
| test "$(git rev-parse FETCH_HEAD)" = "$PAIRED_5829_SHA" | |
| } | |
| git merge-base --is-ancestor "$EVENT_BASE_SHA" "$PAIRED_5829_SHA" | |
| test "$(git diff --name-only "$EVENT_BASE_SHA..$PAIRED_5829_SHA")" = $'src/bench.zig\nsrc/mcp.zig' | |
| git worktree add ../codedb-paired-base "$PAIRED_5829_SHA" | |
| paired_base="../codedb-paired-base" | |
| fi | |
| mkdir -p bench-paired | |
| corpus_source=$(realpath ../codedb-base) | |
| head_source_sha=$(git rev-parse HEAD) | |
| head_tree_sha=$(git rev-parse 'HEAD^{tree}') | |
| git worktree add ../codedb-paired-head "$head_source_sha" | |
| head_cwd=$(realpath ../codedb-paired-head) | |
| run_sample() { | |
| side="$1" | |
| pair="$2" | |
| order="$3" | |
| sequence="$4" | |
| if [[ "$side" == base ]]; then | |
| cwd="$paired_base" | |
| zig_dir="$BASE_ZIG_DIR" | |
| production_sha="$EVENT_BASE_SHA" | |
| else | |
| cwd="$head_cwd" | |
| zig_dir="$GITHUB_WORKSPACE/zig-x86_64-linux-${ZIG_VERSION}" | |
| production_sha="$head_source_sha" | |
| fi | |
| ( | |
| cd "$cwd" | |
| PATH="$zig_dir:$PATH" python3 "$head_cwd/scripts/run-bench-json.py" \ | |
| --bench-side "$side" --bench-pair "$pair" --bench-order "$order" --bench-sequence "$sequence" \ | |
| --production-source-sha "$production_sha" --corpus-source-sha "$EVENT_BASE_SHA" \ | |
| "$GITHUB_WORKSPACE/bench-paired/$side-$(printf '%02d' "$pair").json" \ | |
| --corpus-source "$corpus_source" | |
| ) | |
| } | |
| for ((pair = 1; pair <= PAIRS; pair++)); do | |
| if (( pair % 2 == 1 )); then | |
| run_sample base "$pair" AB 1 | |
| run_sample head "$pair" AB 2 | |
| else | |
| run_sample head "$pair" BA 1 | |
| run_sample base "$pair" BA 2 | |
| fi | |
| done | |
| python3 "$head_cwd/scripts/compare-bench-paired.py" bench-paired --require-parity --require-provenance \ | |
| --allow-parity-skip codedb_snapshot --allow-parity-skip codedb_status --allow-parity-skip codedb_context \ | |
| --expected-head-sha "$head_source_sha" --expected-head-tree-sha "$head_tree_sha" \ | |
| --threshold-pct 10 --min-abs-ns 50000 --markdown-out bench-paired-report.md | |
| - name: Compare | |
| run: | | |
| python3 scripts/compare-bench.py bench-base.json bench-head.json --threshold-pct 10 --markdown-out bench-report.md | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: bench-results | |
| path: | | |
| bench-base.json | |
| bench-head.json | |
| bench-report.md | |
| bench-paired/ | |
| bench-paired-report.md | |
| - name: Publish benchmark summary | |
| if: always() | |
| run: | | |
| report=bench-paired-report.md | |
| [[ -f "$report" ]] || report=bench-report.md | |
| [[ -f "$report" ]] && cat "$report" >> "$GITHUB_STEP_SUMMARY" |