perf(branches): preserve snapshots and reuse verified views #1992

Workflow file for this run

	name: CI

	on:
	pull_request:
	push:
	branches:
	- main
	tags:
	- "v*"
	workflow_dispatch:

	concurrency:
	group: ci-${{ github.workflow }}-${{ github.ref }}
	# Keep main runs alive: their suite result is the post-merge detection
	# channel, and a cancelled job never reaches rust-cache's post-step save.
	# Superseded PR runs are safe to cancel.
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	# The one failpoint feature list `lint` and `test` both compile.
	FAILPOINT_FEATURES: omnigraph-engine/failpoints,omnigraph-cluster/failpoints

	jobs:
	classify_changes:
	name: Classify Changes
	runs-on: ubuntu-latest
	permissions:
	contents: read
	outputs:
	run_full_ci: ${{ steps.filter.outputs.run_full_ci }}
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
	with:
	fetch-depth: 0

	- name: Detect documentation-only changes
	id: filter
	env:
	BEFORE_SHA: ${{ github.event.before }}
	EVENT_NAME: ${{ github.event_name }}
	PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
	PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
	REF_TYPE: ${{ github.ref_type }}
	run: \|
	set -euo pipefail

	if [[ "$EVENT_NAME" == "workflow_dispatch" \|\| "$REF_TYPE" == "tag" ]]; then
	echo "run_full_ci=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	if [[ "$EVENT_NAME" == "pull_request" ]]; then
	base="$PR_BASE_SHA"
	head="$PR_HEAD_SHA"
	else
	base="$BEFORE_SHA"
	head="$GITHUB_SHA"
	if [[ "$base" == "0000000000000000000000000000000000000000" ]]; then
	base="$(git rev-parse "${head}^" 2>/dev/null \|\| true)"
	fi
	fi

	if [[ -z "${base:-}" ]]; then
	echo "run_full_ci=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	# Disable rename collapsing so moving source into docs still reports
	# the source-side deletion and cannot become a docs-only false skip.
	mapfile -t changed < <(git diff --name-only --no-renames "$base" "$head")
	if [[ "${#changed[@]}" -eq 0 ]]; then
	echo "run_full_ci=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	run_full_ci=false
	for path in "${changed[@]}"; do
	case "$path" in
	docs/.md\|docs/.mdx\|docs/.rst\|docs/.adoc) ;;
	README.md\|AGENTS.md\|CLAUDE.md\|CHANGELOG.md\|CONTRIBUTING.md\|CODE_OF_CONDUCT.md) ;;
	LICENSE\|LICENSE.md) ;;
	*)
	run_full_ci=true
	;;
	esac
	done

	printf 'Changed files:\n'
	printf ' %s\n' "${changed[@]}"
	echo "run_full_ci=$run_full_ci" >> "$GITHUB_OUTPUT"

	check_agents_md:
	name: Check AGENTS.md Links
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
	with:
	# The marker check diffs the PR merge commit against its first
	# parent (the current base tip), so both parents must be present.
	fetch-depth: 2

	# Diff-scoped belt over check-docs.py's Markdown scan: flags only
	# conflict-marker lines this pull request ADDS, in any file type, via
	# git's own --check detector (which also respects conflict-marker-size
	# and catches a lone `=======` on added lines). Markers already on the
	# base branch never fail someone else's PR. Runs before the
	# documentation checks so hits are annotated inline even when the
	# Markdown scan would fail the job first.
	- name: Verify the PR adds no merge-conflict markers
	if: github.event_name == 'pull_request'
	run: \|
	set -euo pipefail
	# `git diff --check` exits non-zero on findings, so the pipeline
	# needs the `\|\| true`; grep narrows to markers (drops whitespace
	# complaints). Output shape: path:line: leftover conflict marker
	hits=$(git diff --check HEAD^1 HEAD \| grep 'leftover conflict marker' \|\| true)
	if [ -n "$hits" ]; then
	printf '%s\n' "$hits" \| while IFS=: read -r file line rest; do
	printf '::error file=%s,line=%s::Committed merge-conflict marker\n' "$file" "$line"
	done
	count=$(printf '%s\n' "$hits" \| wc -l \| tr -d ' ')
	echo "::error::this pull request adds $count merge-conflict marker line(s)"
	exit 1
	fi

	- name: Verify AGENTS.md ↔ docs/ cross-links
	run: bash scripts/check-agents-md.sh

	- name: Verify documentation structure and RFC registry
	run: python3 scripts/check-docs.py

	workflow_action_pins:
	name: Check Workflow Action Pins
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Reject mutable external action refs
	run: python3 scripts/check-workflow-action-pins.py

	- name: Enforce release vocabulary gates
	run: python3 scripts/check-release-vocabulary-gates.py

	azure_contract_guards:
	name: Azure Contract Guards
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Verify container and package binary sets match
	run: python3 scripts/check-container-binary-contract.py

	- name: Enforce Azure admission dependency direction
	run: python3 scripts/check-azure-admission-boundary.py

	graph_vocabulary_guard:
	name: Graph Vocabulary Guard
	# Keep this exact reporting context while it remains required by branch
	# protection, but do not run the substrate-sized audit on pull requests.
	# The full audit still runs after merge, on tags, and by manual dispatch.
	if: github.event_name != 'pull_request'
	runs-on: ubuntu-latest
	env:
	# Single switch for the vocabulary audit, disabled by decision on
	# 2026-08-28. Set to "true" to re-enable every audit step at once (the
	# public-Rust tooling and all three checks are gated on it). The job
	# still runs its unit tests and reports success either way, so branch
	# protection and the exact-SHA release gates stay wired.
	VOCABULARY_AUDIT_ENABLED: "false"
	# A cold default + relevant all-features rustdoc pass over all seven public
	# library crates is substrate-sized. The first hosted run completed the
	# public-Rust step in 39 minutes, while the larger breaking-contract tree
	# was still compiling when a 45-minute whole-job bound cancelled it. Keep
	# current/base targets isolated for correctness and retain a finite cold-run
	# ceiling; the pinned-tool and build-target caches make warm runs cheaper.
	timeout-minutes: 75
	permissions:
	contents: read
	steps:
	- name: Checkout source and comparison history
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
	with:
	# `check` reads both the inventory and openapi.json from the exact
	# comparison commit. A shallow checkout could turn that proof into a
	# false missing-base failure.
	fetch-depth: 0

	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Resolve vocabulary comparison base
	id: comparison_base
	env:
	EVENT_NAME: ${{ github.event_name }}
	PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
	PUSH_BEFORE_SHA: ${{ github.event.before }}
	REF_TYPE: ${{ github.ref_type }}
	run: \|
	set -euo pipefail

	zero_sha=0000000000000000000000000000000000000000
	if [[ "$EVENT_NAME" == "pull_request" ]]; then
	base="$PR_BASE_SHA"
	elif [[ "$EVENT_NAME" == "push" && "$REF_TYPE" != "tag" && "$PUSH_BEFORE_SHA" != "$zero_sha" ]]; then
	base="$PUSH_BEFORE_SHA"
	else
	# Tag creation commonly reports an all-zero `before`, while a
	# manual run has no event base. Compare both with the checked-out
	# commit's first parent rather than inventing a mutable branch tip.
	base="$(git rev-parse "${GITHUB_SHA}^")"
	fi

	if [[ -z "${base:-}" \|\| "$base" == "$zero_sha" ]]; then
	echo "::error::unable to resolve a vocabulary comparison base"
	exit 1
	fi
	git cat-file -e "${base}^{commit}"
	echo "sha=$base" >> "$GITHUB_OUTPUT"

	- name: Install public-Rust system dependencies
	if: env.VOCABULARY_AUDIT_ENABLED == 'true'
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev

	- name: Restore pinned public-Rust tooling
	if: env.VOCABULARY_AUDIT_ENABLED == 'true'
	id: public_rust_tool_cache
	uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
	with:
	path: \|
	~/.rustup/toolchains/nightly-2026-08-01-x86_64-unknown-linux-gnu
	~/.rustup/update-hashes/nightly-2026-08-01-x86_64-unknown-linux-gnu
	${{ runner.temp }}/cargo-public-api
	key: vocabulary-public-api-${{ runner.os }}-${{ runner.arch }}-nightly-2026-08-01-cargo-public-api-0.52.0-v1

	- name: Install exact public-Rust tooling
	if: env.VOCABULARY_AUDIT_ENABLED == 'true'
	id: public_rust_tool
	run: \|
	set -euo pipefail

	rustup toolchain install nightly-2026-08-01 --profile minimal
	tool_root="$RUNNER_TEMP/cargo-public-api"
	tool="$tool_root/bin/cargo-public-api"
	if [[ ! -x "$tool" ]] \|\| [[ "$($tool --version)" != "cargo-public-api 0.52.0" ]]; then
	CARGO_INSTALL_ROOT="$tool_root" \
	cargo install cargo-public-api --version 0.52.0 --locked
	fi
	[[ "$($tool --version)" == "cargo-public-api 0.52.0" ]]
	echo "binary=$tool" >> "$GITHUB_OUTPUT"

	- name: Cache public-Rust build data
	if: env.VOCABULARY_AUDIT_ENABLED == 'true'
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: \|
	. -> target/vocabulary-public-api
	key: vocabulary-public-api-nightly-2026-08-01-v1
	save-if: ${{ github.ref == 'refs/heads/main' }}

	- name: Test vocabulary guard
	run: cargo test -p omnigraph-vocabulary-guard --locked

	- name: Check OpenAPI and Rust-string vocabulary
	if: env.VOCABULARY_AUDIT_ENABLED == 'true'
	env:
	BASE_SHA: ${{ steps.comparison_base.outputs.sha }}
	run: \|
	set -euo pipefail
	for surface in openapi rust-string; do
	cargo run -p omnigraph-vocabulary-guard --locked -- \
	check --surface "$surface" --base "$BASE_SHA" \
	--inventory tools/omnigraph-vocabulary-guard/graph-vocabulary-inventory.tsv \
	--openapi openapi.json
	done

	- name: Check public-Rust vocabulary
	if: env.VOCABULARY_AUDIT_ENABLED == 'true'
	env:
	BASE_SHA: ${{ steps.comparison_base.outputs.sha }}
	CARGO_PUBLIC_API: ${{ steps.public_rust_tool.outputs.binary }}
	run: \|
	tool="${CARGO_PUBLIC_API:-cargo-public-api}"
	cargo run -p omnigraph-vocabulary-guard --locked -- \
	check --surface public-rust --base "$BASE_SHA" \
	--inventory tools/omnigraph-vocabulary-guard/graph-vocabulary-inventory.tsv \
	--cargo-public-api "$tool" \
	--public-api-target-dir target/vocabulary-public-api

	release_edge_after_vocabulary:
	name: Release edge after vocabulary audit
	needs: [classify_changes, graph_vocabulary_guard]
	if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.classify_changes.outputs.run_full_ci == 'true'
	permissions:
	actions: read
	contents: write
	uses: ./.github/workflows/release-edge.yml
	with:
	source_ref: ${{ github.sha }}
	ci_run_id: ${{ format('{0}', github.run_id) }}

	release_tag_after_vocabulary:
	name: Release tag after vocabulary audit
	needs: graph_vocabulary_guard
	if: github.event_name == 'push' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v')
	permissions:
	actions: read
	contents: write
	uses: ./.github/workflows/release.yml
	with:
	tag: ${{ github.ref_name }}
	source_ref: ${{ github.ref }}
	ci_run_id: ${{ format('{0}', github.run_id) }}
	secrets: inherit

	publish_tag_image_after_vocabulary:
	name: Publish tag image after vocabulary audit
	needs: graph_vocabulary_guard
	if: github.event_name == 'push' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v')
	permissions:
	actions: read
	contents: read
	packages: write
	uses: ./.github/workflows/publish-image.yml
	with:
	tag: ${{ github.ref_name }}
	source_ref: ${{ github.ref }}
	ci_run_id: ${{ format('{0}', github.run_id) }}
	secrets: inherit

	publish_tag_crates_after_vocabulary:
	name: Publish tag crates after vocabulary audit
	needs: graph_vocabulary_guard
	if: github.event_name == 'push' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v')
	permissions:
	actions: read
	contents: read
	uses: ./.github/workflows/publish-crates.yml
	with:
	tag: ${{ github.ref_name }}
	source_ref: ${{ github.ref }}
	ci_run_id: ${{ format('{0}', github.run_id) }}
	secrets: inherit

	entrypoint_test:
	name: Container Entrypoint
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Verify omnigraph-server entrypoint arg composition
	run: sh docker/entrypoint_test.sh

	- name: Test non-destructive Azure validation mode
	run: sh deploy/azure/validate_test.sh

	- name: Test Azure bootstrap readiness and admission modes
	run: sh deploy/azure/bootstrap-entrypoint_test.sh

	- name: Build Azure bootstrap image from inherited non-root user
	run: sh deploy/azure/bootstrap_image_test.sh

	azure_deployment_validation:
	name: Azure Deployment Validation
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Compile Azure Bicep templates
	# Azure CLI 2.77.0 plus an explicit Bicep version. Both values are
	# immutable inputs; the command performs no Azure login or mutation.
	run: \|
	docker run --rm \
	-v "$PWD:/work" \
	-w /work \
	mcr.microsoft.com/azure-cli:2.77.0@sha256:d41153958a61392a4c9db1591e33b533ca8ab8b59529d212d06bb6ec7e9c1f25 \
	sh -eu -c '
	az bicep install --version v0.46.1 >/dev/null
	az bicep build --file deploy/azure/foundation.bicep --stdout >/dev/null
	az bicep build --file deploy/azure/runtime.bicep --stdout >/dev/null
	'

	fmt:
	name: Format (rustfmt)
	needs: classify_changes
	# Own job so a fmt failure cannot mask the clippy result; fmt compiles
	# nothing, so no dep or cache steps.
	if: github.event_name == 'pull_request' && needs.classify_changes.outputs.run_full_ci == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	# No dtolnay/rust-toolchain action (here or in `lint`): it exports
	# RUSTUP_TOOLCHAIN, overriding the toml pin. Bare install reads the
	# toml; explicit because rustup auto-install has changed across releases.
	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Check formatting
	run: cargo fmt --all --check

	lint:
	name: Lint (clippy)
	needs: classify_changes
	# PR gate. Lints stay `warn` in-tree; only this job denies, so a new
	# toolchain's lints cannot break local builds. Main-push runs exist only
	# to seed the cache (see save-if below).
	if: (github.event_name == 'pull_request' \|\| (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs.classify_changes.outputs.run_full_ci == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 45
	permissions:
	contents: read
	env:
	CARGO_TERM_COLOR: always
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev

	# See the toolchain note in `fmt`. Before the cache step so the cache
	# key sees the pinned rustc.
	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Cache Rust build data
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: \|
	. -> target
	key: lint
	# Save only from main: a save from any other ref (a PR branch, a
	# tag) is restorable by no PR and only evicts shared entries. Save
	# even when the job is red: dependency artifacts are valid whatever
	# the verdict, and a red seed run would otherwise leave every PR
	# cold until main is green again.
	save-if: ${{ github.ref == 'refs/heads/main' }}
	cache-on-failure: true

	- name: Clippy (default features)
	run: cargo clippy --workspace --all-targets --locked -- -D warnings -W clippy::dbg_macro

	# Feature-gated code is invisible to the default build.
	- name: Clippy (failpoints superset)
	run: \|
	cargo clippy --workspace --all-targets --locked \
	--features "$FAILPOINT_FEATURES" \
	-- -D warnings -W clippy::dbg_macro

	test:
	name: Test Workspace
	needs: classify_changes
	# Runs on every non-documentation pull request (fail-fast; a reporting
	# context, not in `.github/branch-protection.json`: under strict checks a
	# required 60-minute context would space merges an hour apart; wait for
	# it before merging) and on main pushes, tags, and workflow_dispatch
	# (`--no-fail-fast`: the post-merge detection channel; the main run also
	# seeds the dependency cache PRs restore). openapi.json is not
	# auto-regenerated: after server/API changes run
	# `OMNIGRAPH_UPDATE_OPENAPI=1 cargo test -p omnigraph-server --test openapi`
	# or the drift check fails this job.
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	runs-on: ubuntu-latest
	# The 2026-07-28 cold diagnostic spent 25m29s on the featureless workspace
	# graph, then another 23m21s compiling the failpoints graph. Build the
	# current tree once with the feature superset instead. The predecessor
	# binary/fence and RustFS shards are independent parallel jobs below, so
	# this job has one current-tree compile and a bounded 60-minute ceiling.
	timeout-minutes: 60
	permissions:
	contents: read
	env:
	CARGO_TERM_COLOR: always
	# Floor for the whole class of debug-build async stack depth, NOT the fix
	# for any one test. Unoptimized builds keep every nested async frame, so
	# a deep engine path (init -> write -> publish -> recovery -> Lance) can
	# approach a default 2 MiB spawned-thread stack. The depth is genuinely
	# environment-sensitive: the same tree overflowed `write_cost` on this
	# Linux runner while macOS stayed under. Individual offenders are still
	# fixed at
	# the source with `Box::pin` (see `helpers::cost::cost_harness`); this
	# keeps CI deterministic across runners while that work continues.
	RUST_MIN_STACK: 16777216
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev

	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Cache Rust build data
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: \|
	. -> target
	# Save only from main, red or green (same rule as `lint`).
	save-if: ${{ github.ref == 'refs/heads/main' }}
	cache-on-failure: true

	# Own step so compile and run wall clock read apart in the log; the
	# run step reuses these binaries (same flags, features, and env).
	- name: Compile the workspace and failpoint test graph
	run: \|
	cargo test --workspace --locked --no-run \
	--features "$FAILPOINT_FEATURES"

	- name: Install genuine v0.9 release for upgrade regression
	run: \|
	set -euo pipefail
	v09_dir="$RUNNER_TEMP/omnigraph-v09"
	# The installer downloads the official archive and verifies its
	# SHA256 before extraction. An exact VERSION never falls back to edge.
	REPO_SLUG=ModernRelay/omnigraph VERSION=v0.9.0 INSTALL_DIR="$v09_dir" \
	bash scripts/install.sh
	test -x "$v09_dir/omnigraph"
	[[ "$("$v09_dir/omnigraph" --version)" == "omnigraph 0.9.0" ]] \
	\|\| { echo "::error::upgrade regression requires the genuine v0.9.0 CLI"; exit 1; }
	echo "OMNIGRAPH_V09_BIN=$v09_dir/omnigraph" >> "$GITHUB_ENV"

	- name: Run workspace and failpoint tests once
	run: \|
	set -euo pipefail
	test_log="$RUNNER_TEMP/workspace-tests.log"
	# A pull request is a gate: stop at the first red suite. Main,
	# tag, and dispatch runs are detection channels: --no-fail-fast,
	# so every independent failure is reported.
	no_fail_fast=()
	if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
	no_fail_fast+=(--no-fail-fast)
	fi
	cargo test --workspace --locked ${no_fail_fast[@]+"${no_fail_fast[@]}"} \
	--features "$FAILPOINT_FEATURES" \
	-- --nocapture 2>&1 \| tee "$test_log"
	if grep -Fq "skipping v0.9 upgrade e2e:" "$test_log"; then
	echo "::error::v0.9 upgrade regression skipped despite a configured release binary"
	exit 1
	fi
	grep -Fq "v0.9 -> v0.10 upgrade e2e completed" "$test_log" \
	\|\| { echo "::error::v0.9 upgrade regression did not complete"; exit 1; }
	grep -Eq "test current_v010_upgrades_genuine_v09_graph_end_to_end \\.\\.\\. ok" "$test_log" \
	\|\| { echo "::error::exact v0.9 upgrade regression did not pass"; exit 1; }


	v5_v6_format_fence:
	name: V5 ↔ V6 Format Fence
	needs: classify_changes
	if: github.event_name != 'pull_request' && needs.classify_changes.outputs.run_full_ci == 'true'
	runs-on: ubuntu-latest
	# This job owns the immutable predecessor build in parallel with the
	# current-tree workspace job. The 2026-07-28 cold measurement was 25m40s
	# for the predecessor CLI; exact current-CLI fence compilation reuses the same locked
	# dependency graph and retains bounded headroom here.
	timeout-minutes: 60
	permissions:
	contents: read
	env:
	CARGO_TERM_COLOR: always
	RUST_MIN_STACK: 16777216
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev

	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Cache cross-version build data
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: \|
	. -> target
	key: crossversion-v5-v6
	save-if: ${{ github.ref == 'refs/heads/main' }}

	# CI pins only the immediate predecessor. Older seams remain env-gated
	# and on-demand, per docs/dev/testing.md.
	- name: Build genuine final-v5 CLI
	env:
	# Final v5-writing main commit immediately before RFC-023 activated
	# the exact non-null `id` primary-key contract in v6. Keep this
	# immutable: HEAD^ is not a stable format reference.
	FINAL_INTERNAL_V5_COMMIT: 46b6d9084fb629b88d4ac9e8c546e0a30d213d19
	run: \|
	set -euo pipefail
	v5_source="$RUNNER_TEMP/omnigraph-final-v5"
	v5_bin="$RUNNER_TEMP/omnigraph-final-v5-bin"
	git fetch --no-tags --depth=1 origin "$FINAL_INTERNAL_V5_COMMIT"
	git worktree add --detach "$v5_source" "$FINAL_INTERNAL_V5_COMMIT"
	cargo build --locked \
	--manifest-path "$v5_source/Cargo.toml" \
	--package omnigraph-cli \
	--bin omnigraph \
	--target-dir "$GITHUB_WORKSPACE/target"
	cp "$GITHUB_WORKSPACE/target/debug/omnigraph" "$v5_bin"
	test -x "$v5_bin"
	# Clean path-package artifacts through both workspace manifests while
	# retaining shared registry dependencies. The predecessor and current
	# packages have the same names+versions, so either manifest alone can
	# miss a stale path identity and make the fence link the wrong rlib.
	cargo clean --workspace --locked \
	--manifest-path "$v5_source/Cargo.toml" \
	--target-dir "$GITHUB_WORKSPACE/target"
	cargo clean --workspace --locked \
	--target-dir "$GITHUB_WORKSPACE/target"
	echo "OMNIGRAPH_V5_BIN=$v5_bin" >> "$GITHUB_ENV"

	- name: Run exact v5↔v6 refusal and rebuild fence
	run: \|
	set -euo pipefail
	test_log="$RUNNER_TEMP/v5-v6-format-fence.log"
	cargo test --locked -p omnigraph-cli --test crossversion_upgrade \
	current_v6_refuses_and_rebuilds_genuine_v5_and_v5_refuses_v6 \
	-- --exact --nocapture 2>&1 \| tee "$test_log"
	if grep -Fq "skipping immediate-predecessor v5 upgrade test" "$test_log"; then
	echo "::error::v5↔v6 format fence skipped despite a configured predecessor binary"
	exit 1
	fi
	grep -Eq "test current_v6_refuses_and_rebuilds_genuine_v5_and_v5_refuses_v6 \.\.\. ok" "$test_log" \
	\|\| { echo "::error::exact v5↔v6 format fence did not execute normally"; exit 1; }
	grep -Eq "test result: ok\. 1 passed; 0 failed" "$test_log" \
	\|\| { echo "::error::v5↔v6 format fence matched the wrong test set"; exit 1; }

	test_aws_feature:
	name: Test omnigraph-server --features aws
	needs: classify_changes
	runs-on: ubuntu-latest
	# Required context whose cache saves only from main: a cold main run
	# that outruns the ceiling is cancelled before the save and every PR
	# stays cold (the dst.yml lesson). Sized for a cold dependency graph.
	timeout-minutes: 60
	permissions:
	contents: read
	env:
	CARGO_TERM_COLOR: always
	steps:
	- name: Skip for documentation-only changes
	if: needs.classify_changes.outputs.run_full_ci != 'true'
	run: echo "Documentation-only change detected; skipping aws feature build."

	- name: Checkout source
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Install system dependencies
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev

	- name: Install pinned toolchain
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	run: rustup toolchain install

	- name: Cache Rust build data
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: \|
	. -> target
	key: aws-feature
	# Save only from main, red or green (same rule as `lint`).
	save-if: ${{ github.ref == 'refs/heads/main' }}
	cache-on-failure: true

	- name: Build omnigraph-server with aws feature
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	run: cargo build --locked -p omnigraph-server --features aws

	- name: Test omnigraph-server with aws feature
	if: needs.classify_changes.outputs.run_full_ci == 'true'
	env:
	RUST_MIN_STACK: 16777216
	run: cargo test --locked -p omnigraph-server --features aws

	rustfs_integration:
	name: RustFS S3 Integration (${{ matrix.shard }})
	# This remains push-/tag-/dispatch-only, but no longer waits for the
	# independent workspace job. Starting both cold compile graphs together
	# removes that dependency from the workflow's critical path.
	#
	# Two parallel feature graphs: the default graph compiles every selected
	# engine/server/cluster/CLI test binary in one Cargo invocation, while the
	# failpoints graph stays isolated. Presenting the full default DAG together
	# avoids discovering it as a serial compile/link waterfall. The previous
	# five-shard layout repeated the same ~26-29m cold substrate compile five
	# times and maintained five target caches, pressuring GitHub's repository
	# cache cap. `fail-fast: false` lets both graphs report independently.
	needs: classify_changes
	if: github.event_name != 'pull_request' && needs.classify_changes.outputs.run_full_ci == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 45
	strategy:
	fail-fast: false
	matrix:
	shard:
	- default
	- failpoints
	permissions:
	contents: read
	env:
	AWS_ACCESS_KEY_ID: omnigraphci
	AWS_SECRET_ACCESS_KEY: omnigraphci-secret
	AWS_REGION: us-east-1
	AWS_ENDPOINT_URL: http://127.0.0.1:9000
	AWS_ENDPOINT_URL_S3: http://127.0.0.1:9000
	AWS_ALLOW_HTTP: "true"
	AWS_S3_FORCE_PATH_STYLE: "true"
	OMNIGRAPH_S3_TEST_BUCKET: omnigraph-ci
	OMNIGRAPH_S3_TEST_PREFIX: github-actions
	CARGO_TERM_COLOR: always
	RUST_MIN_STACK: 16777216
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev python3-pip

	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Cache Rust build data
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	# One cache per feature graph avoids both feature thrash and the
	# previous five copies of the shared substrate.
	key: ${{ matrix.shard }}
	workspaces: \|
	. -> target
	save-if: ${{ github.ref == 'refs/heads/main' }}

	- name: Start RustFS
	# Pinned to 1.0.0-beta.12 (2026-07-30). beta.8 carried hyper's short
	# header_read_timeout for idle keep-alive connections (rustfs#4360,
	# fixed in beta.10), matching the intermittent "error sending request"
	# PUT failures observed here on pooled connections. The S3 suites
	# (engine s3_storage, server s3, cluster s3_cluster, engine failpoints
	# s3_) were validated against the beta.12
	# binary locally before this bump. Keep CI pinned to the immutable
	# multi-arch index digest so upgrades are deliberate.
	run: \|
	docker rm -f rustfs >/dev/null 2>&1 \|\| true
	docker run -d \
	--name rustfs \
	-p 9000:9000 \
	-p 9001:9001 \
	-e RUSTFS_ACCESS_KEY="${AWS_ACCESS_KEY_ID}" \
	-e RUSTFS_SECRET_KEY="${AWS_SECRET_ACCESS_KEY}" \
	rustfs/rustfs:1.0.0-beta.12@sha256:41fe89380f4120a337790c02af192c3fe7bb55c3edc2e6e9357b487b47c6ab21 \
	/data

	- name: Install AWS CLI
	run: \|
	python3 -m pip install --user awscli
	echo "$HOME/.local/bin" >> "$GITHUB_PATH"

	- name: Create RustFS test bucket
	run: \|
	set -euo pipefail
	rustfs_ready=false
	for attempt in $(seq 1 30); do
	if aws --endpoint-url "${AWS_ENDPOINT_URL_S3}" s3api list-buckets >/dev/null 2>&1; then
	rustfs_ready=true
	break
	fi
	echo "RustFS readiness check ${attempt}/30 failed; retrying"
	sleep 2
	done

	if [ "$rustfs_ready" != true ]; then
	echo "::error::RustFS did not become ready after 30 attempts"
	exit 1
	fi

	aws --endpoint-url "${AWS_ENDPOINT_URL_S3}" \
	s3api create-bucket \
	--bucket "${OMNIGRAPH_S3_TEST_BUCKET}"

	# Compile and execute the selected default targets in one Cargo resolve.
	# A superset prebuild followed by narrower commands would still rebuild
	# workspace crates under different feature fingerprints.
	- name: Run RustFS default integration graph
	if: matrix.shard == 'default'
	run: \|
	set -euo pipefail
	test_log="$RUNNER_TEMP/rustfs-default.log"
	cargo test --locked \
	-p omnigraph-engine \
	-p omnigraph-server \
	-p omnigraph-cluster \
	-p omnigraph-cli \
	--test s3_storage \
	--test lance_surface_guards \
	--test s3 \
	--test s3_cluster \
	--test system_local \
	-- --nocapture --test-threads=1 2>&1 \| tee "$test_log"

	if grep -Eiq \
	"SKIP public_physical_ref_token_rejects_s3_same_version_aba\|skipping s3 " \
	"$test_log"
	then
	echo "::error::a configured RustFS test emitted its unconfigured skip"
	exit 1
	fi

	for test_name in \
	s3_compatible_graph_lifecycle_works \
	s3_branch_change_merge_flow_works \
	s3_public_load_uses_hidden_run_and_publishes \
	s3_adapter_conditional_writes_contract \
	s3_schema_apply_migrates_live_graph \
	s3_fresh_branch_traversal_reuses_main_graph_index_with_etags \
	public_physical_ref_token_rejects_s3_same_version_aba \
	server_opens_s3_graph_directly_and_serves_snapshot_and_read \
	server_boots_cluster_from_bare_storage_uri_and_serves_query \
	s3_cluster_full_lifecycle_import_apply_serve_evolve_delete \
	local_cli_s3_end_to_end_init_load_read_flow
	do
	grep -Eq "test ${test_name} \.\.\. ok" "$test_log" \
	\|\| { echo "::error::required RustFS cell '${test_name}' did not execute normally"; exit 1; }
	done

	# NOTE: the RFC-013 step-3a data-table opener COST gate (write_cost_s3) used
	# to run here. It is a deterministic IO-count gate, not a correctness test —
	# performance/cost contracts belong in a dedicated perf harness on a stable
	# runner + own cadence, not on the every-merge correctness path. Moved out of
	# CI pending that harness; run it on demand with a bucket set:
	# OMNIGRAPH_S3_TEST_BUCKET=… cargo test -p omnigraph-engine --test write_cost_s3

	- name: Run RustFS recovery-sidecar lifecycle
	if: matrix.shard == 'failpoints'
	# Sidecar put/list/delete through the S3 storage backend on a
	# real bucket (the failpoint only wedges the publisher; the
	# sidecar I/O is exercised for real). Name filter `s3_` matches
	# the bucket-gated tests in the failpoints target only; the
	# grep guards against the filter going vacuous (cargo passes
	# with 0 tests matched) if those tests are ever renamed.
	run: \|
	set +e
	output=$(cargo test --locked -p omnigraph-engine --features failpoints --test failpoints s3_ -- --nocapture 2>&1); status=$?
	set -e
	echo "$output"
	[ "$status" -eq 0 ] \|\| exit "$status"
	echo "$output" \| grep -Eq "test result: ok\. [1-9][0-9]* passed" \
	\|\| { echo "::error::filter 's3_' matched no tests — vacuous pass"; exit 1; }

	- name: Dump RustFS diagnostics on failure
	if: failure()
	run: \|
	set +e
	echo "::group::docker inspect rustfs"
	# Limit inspection to runtime state: full inspect output includes the
	# container environment and would disclose the test credentials.
	docker inspect --format 'State={{json .State}} Image={{.Image}}' rustfs
	echo "::endgroup::"

	echo "::group::docker logs rustfs"
	docker logs rustfs
	echo "::endgroup::"

	diagnostics_dir=$(mktemp -d "${RUNNER_TEMP}/rustfs-logs.XXXXXX")
	echo "::group::RustFS /logs"
	if docker cp rustfs:/logs "$diagnostics_dir"; then
	find "$diagnostics_dir" -type f -exec sh -c '
	for log_file do
	echo "===== ${log_file} ====="
	tail -n 2000 "$log_file"
	done
	' sh {} +
	else
	echo "Unable to copy /logs from the RustFS container"
	fi
	echo "::endgroup::"

	azurite_integration:
	name: Azurite Azure Integration
	# This configured backend contract runs after merge, on tags, and by
	# manual dispatch. Keep it independent of `test` so cold backend graphs
	# start together and a failure reports at its real boundary.
	needs: classify_changes
	if: github.event_name != 'pull_request' && needs.classify_changes.outputs.run_full_ci == 'true'
	runs-on: ubuntu-latest
	# Cold failpoint and default-feature Lance graphs compile sequentially on
	# the first cache miss; keep the timeout above the RustFS shard envelope.
	timeout-minutes: 90
	permissions:
	contents: read
	env:
	AZURE_STORAGE_USE_EMULATOR: "true"
	AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1
	AZURITE_BLOB_STORAGE_URL: http://127.0.0.1:10000
	OMNIGRAPH_AZURE_TEST_CONTAINER: omnigraph-tests
	CARGO_TERM_COLOR: always
	RUST_MIN_STACK: 16777216
	steps:
	- name: Checkout source
	uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler libprotobuf-dev

	- name: Install pinned toolchain
	run: rustup toolchain install

	- name: Cache Azure integration build data
	uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: \|
	. -> target
	key: azurite-azure
	save-if: ${{ github.ref == 'refs/heads/main' }}

	- name: Start pinned Azurite Blob service
	# Azurite 3.35.0, pinned to the immutable multi-arch index digest.
	run: \|
	set -euo pipefail
	docker rm -f azurite >/dev/null 2>&1 \|\| true
	docker run -d \
	--name azurite \
	-p 10000:10000 \
	mcr.microsoft.com/azure-storage/azurite:3.35.0@sha256:647c63a91102a9d8e8000aab803436e1fc85fbb285e7ce830a82ee5d6661cf37 \
	azurite-blob --blobHost 0.0.0.0 --blobPort 10000
	for _ in $(seq 1 60); do
	if curl --silent --show-error \
	"${AZURITE_BLOB_STORAGE_URL}/${AZURE_STORAGE_ACCOUNT_NAME}?comp=list" \
	>/dev/null
	then
	exit 0
	fi
	sleep 1
	done
	echo "::error::Azurite did not become reachable"
	docker logs azurite
	exit 1

	- name: Create Azurite test container
	# This is the public emulator key, never a production credential. The
	# emulator canonicalized resource repeats the account path segment.
	run: \|
	python3 - <<'PY'
	import base64
	import datetime
	import email.utils
	import hashlib
	import hmac
	import os
	import urllib.request

	account = os.environ['AZURE_STORAGE_ACCOUNT_NAME']
	container = os.environ['OMNIGRAPH_AZURE_TEST_CONTAINER']
	endpoint = os.environ['AZURITE_BLOB_STORAGE_URL']
	key = base64.b64decode(
	'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/'
	'K1SZFPTOtr/KBHBeksoGMGw=='
	)
	date = email.utils.format_datetime(
	datetime.datetime.now(datetime.timezone.utc), usegmt=True
	)
	version = '2023-11-03'
	canonical_headers = f'x-ms-date:{date}\nx-ms-version:{version}\n'
	canonical_resource = (
	f'/{account}/{account}/{container}\nrestype:container'
	)
	string_to_sign = (
	'PUT\n\n\n\n\n\n\n\n\n\n\n\n'
	+ canonical_headers
	+ canonical_resource
	)
	signature = base64.b64encode(
	hmac.new(key, string_to_sign.encode(), hashlib.sha256).digest()
	).decode()
	request = urllib.request.Request(
	f'{endpoint}/{account}/{container}?restype=container',
	method='PUT',
	headers={
	'Authorization': f'SharedKey {account}:{signature}',
	'Content-Length': '0',
	'x-ms-date': date,
	'x-ms-version': version,
	},
	)
	with urllib.request.urlopen(request, timeout=10) as response:
	if response.status != 201:
	raise SystemExit(f'unexpected create-container status: {response.status}')
	PY

	- name: Run exact Azure integration owners
	run: \|
	set -euo pipefail
	test_log="$RUNNER_TEMP/azurite-azure.log"
	: >"$test_log"

	cargo test --locked -p omnigraph-storage \
	tests::contract_suite_azure_when_configured \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-azure-admission \
	tests::configured_azurite_enforces_one_lease_owner_and_explicit_break \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-azure-admission --test supervisor \
	-- --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-engine --features failpoints --test failpoints \
	azure_recovery_rolls_forward_after_finalize_publisher_failure \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-engine --features failpoints --test failpoints \
	azure_schema_apply_recovers_source_and_destination_after_partial_rename \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-cluster --test s3_cluster \
	azure_cluster_full_lifecycle_import_apply_serve_evolve_delete \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-server --test s3 \
	server_opens_azure_graph_directly_and_serves_snapshot_and_read \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-server --test s3 \
	server_boots_azure_cluster_from_bare_storage_uri_and_serves_query \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"
	cargo test --locked -p omnigraph-cli --test system_local \
	local_cli_azure_end_to_end_init_load_read_flow \
	-- --exact --nocapture 2>&1 \| tee -a "$test_log"

	if grep -Eiq 'skipping .azure\|skipping .azurite' "$test_log"; then
	echo "::error::a configured Azure test emitted its unconfigured skip"
	exit 1
	fi
	for test_name in \
	contract_suite_azure_when_configured \
	configured_azurite_enforces_one_lease_owner_and_explicit_break \
	held_admission_never_launches_child_then_runs_once_after_release \
	held_job_terminated_before_admission_fails_without_launching_child \
	graceful_server_signal_drains_process_group_before_release \
	unexpected_server_exit_strands_lease_until_explicit_break \
	azure_recovery_rolls_forward_after_finalize_publisher_failure \
	azure_schema_apply_recovers_source_and_destination_after_partial_rename \
	azure_cluster_full_lifecycle_import_apply_serve_evolve_delete \
	server_opens_azure_graph_directly_and_serves_snapshot_and_read \
	server_boots_azure_cluster_from_bare_storage_uri_and_serves_query \
	local_cli_azure_end_to_end_init_load_read_flow
	do
	grep -Eq "test (tests::)?${test_name} \.\.\. ok" "$test_log" \
	\|\| { echo "::error::required Azure cell '${test_name}' did not execute normally"; exit 1; }
	done

	- name: Verify control and Lance objects share the declared container
	run: \|
	python3 - "$RUNNER_TEMP/azurite-inventory.txt" <<'PY'
	import base64
	import datetime
	import email.utils
	import hashlib
	import hmac
	import os
	import pathlib
	import urllib.request
	import xml.etree.ElementTree as ET

	account = os.environ['AZURE_STORAGE_ACCOUNT_NAME']
	container = os.environ['OMNIGRAPH_AZURE_TEST_CONTAINER']
	endpoint = os.environ['AZURITE_BLOB_STORAGE_URL']
	key = base64.b64decode(
	'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/'
	'K1SZFPTOtr/KBHBeksoGMGw=='
	)
	date = email.utils.format_datetime(
	datetime.datetime.now(datetime.timezone.utc), usegmt=True
	)
	version = '2023-11-03'
	canonical_headers = f'x-ms-date:{date}\nx-ms-version:{version}\n'
	canonical_resource = (
	f'/{account}/{account}/{container}\ncomp:list\nrestype:container'
	)
	string_to_sign = (
	'GET\n\n\n\n\n\n\n\n\n\n\n\n'
	+ canonical_headers
	+ canonical_resource
	)
	signature = base64.b64encode(
	hmac.new(key, string_to_sign.encode(), hashlib.sha256).digest()
	).decode()
	request = urllib.request.Request(
	f'{endpoint}/{account}/{container}?comp=list&restype=container',
	headers={
	'Authorization': f'SharedKey {account}:{signature}',
	'x-ms-date': date,
	'x-ms-version': version,
	},
	)
	with urllib.request.urlopen(request, timeout=10) as response:
	inventory = ET.fromstring(response.read())
	if inventory.findtext('NextMarker'):
	raise SystemExit('Azure inventory exceeded one explicit 5,000-object page')
	names = [node.text or '' for node in inventory.findall('.//Name')]
	pathlib.Path(__import__('sys').argv[1]).write_text('\n'.join(names) + '\n')
	if not any('/__cluster/' in name for name in names):
	raise SystemExit('no Azure cluster control object was observed')
	if not any('.lance/' in name or '/_versions/' in name for name in names):
	raise SystemExit('no Azure Lance dataset object was observed')
	if not any(name.startswith('__omnigraph_azure_admission/v1/') for name in names):
	raise SystemExit('no root-derived admission object was observed')
	PY

	- name: Upload Azurite integration evidence
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: azurite-azure-integration-${{ github.run_id }}-${{ github.run_attempt }}
	path: \|
	${{ runner.temp }}/azurite-azure.log
	${{ runner.temp }}/azurite-inventory.txt
	if-no-files-found: warn
	retention-days: 14

	- name: Dump Azurite logs on failure
	if: failure()
	run: docker logs azurite

Navigation Menu

Uh oh!

perf(branches): preserve snapshots and reuse verified views #1992

Workflow file

perf(branches): preserve snapshots and reuse verified views #1992

Uh oh!

Workflow file for this run