feat(wal): linked-SQE io_uring group commit #233

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	pull_request:

	# Deny-all by default; each job re-grants only what it needs.
	permissions: {}

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	CARGO_TERM_COLOR: always
	RUSTFLAGS: -D warnings

	jobs:
	markdown-format:
	name: prettier markdown
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
	- run: bunx prettier --check '*/.md'

	gpudirect-check:
	name: cargo check (rdma + gpudirect)
	runs-on: ubuntu-latest
	timeout-minutes: 30
	permissions:
	contents: read
	container:
	image: nvidia/cuda:12.5.0-devel-ubuntu22.04
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- name: Install build deps
	run: \|
	apt-get update
	apt-get install -y --no-install-recommends \
	curl build-essential pkg-config clang \
	libibverbs-dev libnuma-dev ca-certificates \
	python3 python3-dev
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: gpudirect-check
	- run: cargo check -p aether-stream --tests --features "rdma gpudirect"
	- run: cargo check -p aethergraph-py --features gpudirect

	rust:
	name: rust ${{ matrix.label }}
	runs-on: ${{ matrix.os }}
	timeout-minutes: 30
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: macos-latest
	label: macos-stable-default
	features: ""
	run_rdma: false
	- os: ubuntu-latest
	label: ubuntu-stable-default
	features: ""
	run_rdma: false
	- os: ubuntu-latest
	label: ubuntu-stable-rdma
	features: "rdma"
	run_rdma: true

	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	components: clippy, rustfmt
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: ${{ matrix.label }}

	- name: Install ubuntu system deps
	if: matrix.os == 'ubuntu-latest'
	run: \|
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends \
	clang libbpf-dev pkg-config libibverbs-dev libnuma-dev

	- name: Load SoftRoCE (rxe) on loopback
	if: matrix.run_rdma
	run: \|
	sudo apt-get install -y --no-install-recommends "linux-modules-extra-$(uname -r)" \|\| true
	if sudo modprobe rdma_rxe 2>/dev/null && sudo rdma link add rxe0 type rxe netdev lo; then
	rdma link show
	else
	echo "rxe unavailable on this runner; RDMA e2e tests self-skip"
	fi

	- name: rustfmt
	run: cargo fmt --all -- --check

	- name: clippy (no features)
	run: cargo clippy --workspace --all-targets --no-deps -- -D warnings

	- name: clippy (with features)
	if: matrix.features != ''
	run: \|
	cargo clippy --workspace --all-targets --no-deps \
	--features "${{ matrix.features }}" -- -D warnings

	- name: xdp_bpf compile check
	if: matrix.os == 'ubuntu-latest'
	run: cargo check -p aether-stream --tests --features xdp_bpf

	- name: mlx5dv compile check
	if: matrix.run_rdma
	run: cargo check -p aether-stream --tests --features mlx5dv

	- name: cargo test --lib
	run: \|
	if [ -n "${{ matrix.features }}" ]; then
	cargo test --workspace --lib --features "${{ matrix.features }}"
	else
	cargo test --workspace --lib
	fi

	- name: cargo test --tests
	run: \|
	if [ -n "${{ matrix.features }}" ]; then
	cargo test --workspace --tests --features "${{ matrix.features }}"
	else
	cargo test --workspace --tests
	fi

	- name: cargo check --benches
	run: cargo check --workspace --benches

	- name: bench smoke
	if: matrix.label == 'ubuntu-stable-default'
	run: cargo bench --workspace --no-fail-fast -- --quick --noplot

	- name: Coverage summary
	if: always()
	run: \|
	features="${{ matrix.features }}"
	[ -z "$features" ] && features="(none)"
	ts=$(find target/debug/deps -maxdepth 1 -type f \
	-name '-' \
	! -name '*.d' \
	! -name '*.rlib' \
	! -name '*.rmeta' \
	2>/dev/null \| wc -l \| tr -d ' ')
	{
	echo "## ${{ matrix.label }}"
	echo ""
	echo "\| metric \| value \|"
	echo "\|---\|---\|"
	echo "\| os \| ${{ matrix.os }} \|"
	echo "\| features \| $features \|"
	echo "\| compiled test binaries \| $ts \|"
	} >> "$GITHUB_STEP_SUMMARY"

	miri:
	name: miri unsafe-code interpreter
	runs-on: ubuntu-latest
	timeout-minutes: 30
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: nightly
	components: miri
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: miri
	- run: cargo +nightly miri test -p aether-graph --test miri_smoke
	env:
	MIRIFLAGS: -Zmiri-strict-provenance
	- run: cargo +nightly miri test -p aether-mem --test miri_smoke
	env:
	MIRIFLAGS: -Zmiri-strict-provenance
	- name: Summary
	if: always()
	run: \|
	{
	echo "## miri"
	echo ""
	echo "Ran \`aether-graph::tests::miri_smoke\` under \`cargo miri\` with"
	echo "\`-Zmiri-strict-provenance\`. Covers Arena, Chunk, CTree, and"
	echo "\`DynamicGraph::Writer\` paths end-to-end."
	} >> "$GITHUB_STEP_SUMMARY"

	wal:
	name: wal recovery
	runs-on: ubuntu-latest
	timeout-minutes: 20
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: wal
	- run: cargo test -p aether-graph --features wal
	- name: wal over io_uring group commit
	run: cargo test -p aether-graph --features "wal io-uring"
	- name: Summary
	if: always()
	run: \|
	{
	echo "## wal"
	echo ""
	echo "Round-trip + crash-recovery tests for the append-only WAL,"
	echo "portable path and linked-SQE io_uring group-commit path."
	} >> "$GITHUB_STEP_SUMMARY"

	loom:
	name: loom model checker
	runs-on: ubuntu-latest
	timeout-minutes: 30
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: loom
	- run: cargo test -p aether-graph --features loom --test loom_writer_guard
	- run: cargo test -p aethergraph-core --features loom --test loom_union_find
	- run: cargo test -p aether-mem --test loom_lockfree --release
	env:
	RUSTFLAGS: --cfg loom
	- name: Summary
	if: always()
	run: \|
	{
	echo "## loom"
	echo ""
	echo "Exhaustive interleaving checks of the writer-guard CAS, the lock-free union-find, and the free-list."
	} >> "$GITHUB_STEP_SUMMARY"

	supply-chain:
	name: cargo-deny + audit
	runs-on: ubuntu-latest
	timeout-minutes: 20
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: supply-chain
	- name: Install cargo-deny + cargo-audit
	run: cargo install --locked cargo-deny cargo-audit
	- name: cargo deny check
	run: cargo deny check
	- name: cargo audit
	# Mirror `[advisories.ignore]` in deny.toml — cargo-audit has no local config.
	run: cargo audit --ignore RUSTSEC-2024-0436

	unused-deps:
	name: Unused Dependencies
	runs-on: ubuntu-latest
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- name: Install cargo-machete
	uses: taiki-e/install-action@6a1bd70eaac3c8bdf093356838d7ee09fda951cf # v2.85.5
	with:
	tool: cargo-machete
	- name: Check for unused dependencies
	run: cargo machete

	fuzz:
	name: cargo-fuzz short budget
	runs-on: ubuntu-latest
	timeout-minutes: 8
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: nightly
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: fuzz
	- run: cargo install cargo-fuzz --locked
	- name: Verify corpus is reproducible
	working-directory: fuzz
	run: cargo test --test seed_determinism
	- name: Regenerate fixed-seed corpus
	working-directory: fuzz
	run: cargo run --bin seed_corpus
	- name: ctree_insert_sequences (60s)
	working-directory: fuzz
	run: cargo +nightly fuzz run ctree_insert_sequences -- -max_total_time=60
	- name: csr_loader_bytes (60s)
	working-directory: fuzz
	run: cargo +nightly fuzz run csr_loader_bytes -- -max_total_time=60

	python:
	name: python ${{ matrix.os }}
	runs-on: ${{ matrix.os }}
	timeout-minutes: 30
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest]
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
	with:
	enable-cache: true
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: python-${{ matrix.os }}
	- name: Sync test deps
	# uv 0.11+ build isolation drops cargo off PATH; install deps without
	# the project, then build the wheel without re-syncing.
	working-directory: python
	run: uv sync --group test --group dev --no-install-project
	- name: Build extension
	working-directory: python
	run: uv run --no-sync maturin develop --release
	- name: Run pytest
	working-directory: python
	timeout-minutes: 10
	env:
	PYTHONUNBUFFERED: "1"
	run: uv run pytest tests/ -v --tb=short -o faulthandler_timeout=300

	python-wheel:
	name: wheel ${{ matrix.level }}
	runs-on: ubuntu-latest
	timeout-minutes: 30
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	include:
	- level: x86-64
	rustflags: -D warnings
	out_dir: dist-x86-64
	- level: x86-64-v3
	rustflags: -D warnings -C target-cpu=x86-64-v3
	out_dir: dist-x86-64-v3
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
	with:
	enable-cache: true
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: wheel-${{ matrix.level }}
	- name: Sync build deps
	working-directory: python
	run: uv sync --group dev --no-install-project
	- name: Build wheel
	working-directory: python
	run: uv run --no-sync maturin build --release --out ${{ matrix.out_dir }}
	env:
	RUSTFLAGS: ${{ matrix.rustflags }}
	- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: aethergraph-wheel-${{ matrix.level }}
	path: python/${{ matrix.out_dir }}/*.whl

	python-wheel-pgo:
	name: wheel x86-64 pgo
	runs-on: ubuntu-latest
	timeout-minutes: 45
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	components: llvm-tools
	- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
	with:
	enable-cache: true
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: wheel-pgo
	- name: Install BOLT
	run: \|
	sudo apt-get update
	if sudo apt-get install -y bolt-18; then
	# Reached through the real LLVM bindir, not a symlink into another
	# prefix: llvm-bolt finds its runtime archive relative to itself.
	dirname "$(readlink -f "$(command -v llvm-bolt-18)")" >> "$GITHUB_PATH"
	else
	echo "bolt-18 unavailable; building PGO-only wheel"
	fi
	- name: Build PGO wheel
	run: scripts/pgo-wheel.sh
	- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: aethergraph-wheel-x86-64-pgo
	path: python/dist-pgo/*.whl

	rdma-hardware:
	name: rdma hardware (self-hosted)
	# TODO(infra): wire to a self-hosted Linux runner with rdma_rxe loaded
	# and `aether-test-rdma` label. Until then this job is opt-in via
	# workflow_dispatch and skips on PR.
	if: \|
	github.event_name == 'workflow_dispatch' \|\|
	contains(github.event.head_commit.message, '[rdma]')
	runs-on: [self-hosted, linux, aether-test-rdma]
	timeout-minutes: 40
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	key: rdma-hw
	- name: Verify rxe module
	run: ip link show rxe0 \|\| (sudo modprobe rdma_rxe && sudo rdma link add rxe0 type rxe netdev lo)
	- name: Run gated RDMA tests
	run: \|
	cargo test -p aether-stream --features rdma --tests -- --ignored
	env:
	AETHER_SKIP_RDMA: ""

	# ── Lint the workflow files themselves (YAML + embedded shell) ──
	# shellcheck ships on the runner, so actionlint also checks the inline
	# bash. Catches dead runner labels, quoting bugs, and bad expressions
	# before they break a tag release (where failures are most expensive).
	actionlint:
	name: actionlint
	runs-on: ubuntu-latest
	timeout-minutes: 5
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false

	- name: Run actionlint (SHA-pinned download + checksum)
	run: \|
	VER="1.7.12"
	SHA="8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8"
	curl -fsSL "https://github.com/rhysd/actionlint/releases/download/v${VER}/actionlint_${VER}_linux_amd64.tar.gz" -o actionlint.tgz
	echo "${SHA} actionlint.tgz" \| sha256sum -c -
	tar -xzf actionlint.tgz actionlint
	./actionlint

	summary:
	name: coverage summary
	runs-on: ubuntu-latest
	timeout-minutes: 5
	permissions:
	contents: read
	needs: [markdown-format, gpudirect-check, rust, miri, loom, wal, supply-chain, unused-deps, python, python-wheel, python-wheel-pgo]
	if: always()
	steps:
	- name: Render summary
	run: \|
	{
	echo "# AetherGraph CI coverage"
	echo ""
	echo "\| job \| platform \| features \| result \|"
	echo "\|---\|---\|---\|---\|"
	echo "\| markdown-format \| Linux x86_64 \| prettier \| ${{ needs.markdown-format.result }} \|"
	echo "\| gpudirect-check \| Linux x86_64 \| rdma + gpudirect (cargo check) \| ${{ needs.gpudirect-check.result }} \|"
	echo "\| rust macos-stable-default \| macOS arm64 \| (none) \| ${{ needs.rust.result }} \|"
	echo "\| rust ubuntu-stable-default \| Linux x86_64 \| (none) \| ${{ needs.rust.result }} \|"
	echo "\| rust ubuntu-stable-rdma \| Linux x86_64 \| rdma \| ${{ needs.rust.result }} \|"
	echo "\| miri \| Linux x86_64 \| nightly + miri \| ${{ needs.miri.result }} \|"
	echo "\| loom \| Linux x86_64 \| loom feature \| ${{ needs.loom.result }} \|"
	echo "\| wal \| Linux x86_64 \| wal feature \| ${{ needs.wal.result }} \|"
	echo "\| python ubuntu \| Linux x86_64 \| test group \| ${{ needs.python.result }} \|"
	echo "\| python macos \| macOS arm64 \| test group \| ${{ needs.python.result }} \|"
	echo "\| wheel x86-64 \| Linux x86_64 \| portable baseline \| ${{ needs.python-wheel.result }} \|"
	echo "\| wheel x86-64-v3 \| Linux x86_64 \| target-cpu=x86-64-v3 \| ${{ needs.python-wheel.result }} \|"
	echo "\| wheel x86-64 pgo \| Linux x86_64 \| PGO + BOLT \| ${{ needs.python-wheel-pgo.result }} \|"
	echo ""
	echo "Hardware-gated tests (SoftRoCE micro-bench, SRD loopback,"
	echo "billion-node scale) run only on the self-hosted \`rdma-hardware\`"
	echo "job (opt-in via \`[rdma]\` in commit message or workflow_dispatch)."
	} >> "$GITHUB_STEP_SUMMARY"

	# TODO(bench): criterion job that publishes flamegraphs + numbers to
	# gh-pages for side-by-side comparison vs PyG / DGL. Skeleton lives in
	# `crates/aethergraph-core/benches/`; the missing piece is a runner pinned
	# to a quiet machine class so benchmark noise stays comparable across runs.

Navigation Menu

feat(wal): linked-SQE io_uring group commit #233

Workflow file

feat(wal): linked-SQE io_uring group commit #233

Uh oh!

Workflow file for this run