feat(runtime): report execution choice points #252

Workflow file for this run

	name: PR

	# The only CI a pull request runs. Pushes to main and tags are covered by the
	# CircleCI config; this workflow mirrors its build-test workflow, not release.
	on:
	pull_request:

	concurrency:
	group: pr-${{ github.event.pull_request.number }}
	cancel-in-progress: true

	env:
	# The OMG training corpus is not vendored. Requiring it here turns "corpus
	# absent" into a test failure instead of a skip, so the gate cannot pass green
	# without actually running (see internal/core/model/training_examples_test.go).
	OPENSYSML_REQUIRE_TRAINING_CORPUS: "1"
	# Same for the three pinned OMG pilot corpora, which are downloaded below and
	# gated by internal/core/model/pilot_corpora_test.go.
	OPENSYSML_REQUIRE_PILOT_CORPORA: "1"
	# z3 is installed below, so "no solver, therefore skip" would exercise nothing:
	# this turns an absent solver into a failure (see internal/core/solve).
	OPENSYSML_REQUIRE_SMT: "1"

	jobs:
	# Which areas the pull request touches. A path no area claims turns every area
	# on, so a new directory is over-tested rather than untested.
	changes:
	name: Changed areas
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 5
	outputs:
	go: ${{ steps.areas.outputs.go }}
	docs: ${{ steps.areas.outputs.docs }}
	node: ${{ steps.areas.outputs.node }}
	python: ${{ steps.areas.outputs.python }}
	java: ${{ steps.areas.outputs.java }}
	rust: ${{ steps.areas.outputs.rust }}
	vscode: ${{ steps.areas.outputs.vscode }}
	steps:
	- name: Check out repository
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Test the area filter
	run: scripts/ci-changed-areas-test.sh

	- name: Decide which areas to run
	id: areas
	env:
	BASE_REF: ${{ github.base_ref }}
	run: scripts/ci-changed-areas.sh "origin/$BASE_REF" HEAD \| tee -a "$GITHUB_OUTPUT"

	race-tests:
	name: Go race tests
	needs: changes
	if: needs.changes.outputs.go == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 40
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod
	cache: true

	- name: Download Go modules
	run: go mod download

	# Pinned by digest, so a substituted asset is not run. The release binary rather
	# than `go run` keeps buf's own module tree out of the checksum-database path.
	- name: Install buf
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/buf \
	https://github.com/bufbuild/buf/releases/download/v1.57.2/buf-Linux-x86_64
	echo "644e6fa4a62afc2b0d425d3ac9538d2f512947c0f2e82fc95c41a5eab6729414 /tmp/buf" \| sha256sum --check --strict
	sudo install -m 0755 /tmp/buf /usr/local/bin/buf
	buf --version

	# One buf template generates both the Go stubs and the Java client's message
	# classes, so without this check the committed stubs can drift from the schema.
	# `git add -N` puts a regenerated file the commit had deleted into the diff.
	- name: Verify the committed Go and Java stubs are current
	run: \|
	make proto-buf BUF=buf
	git add -N api/proto \
	clients/java/opensysml-client/src/main/java/org/openmbee/opensysml/proto
	git diff --exit-code api/proto \
	clients/java/opensysml-client/src/main/java/org/openmbee/opensysml/proto

	# The solver is an external process, never linked in: the solver-dependent
	# tests need one on PATH to run rather than skip.
	- name: Install Z3
	run: \|
	sudo apt-get update
	sudo apt-get install -y z3
	z3 --version

	# The second verified backend, for the portability gate below: cvc5 has no
	# apt package, so the official static release is what CI runs, pinned by
	# digest so a substituted asset is not run.
	- name: Install cvc5
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/cvc5.zip \
	https://github.com/cvc5/cvc5/releases/download/cvc5-1.3.4/cvc5-Linux-x86_64-static.zip
	echo "dcdbfada0ce493ee98259c0816e0daafc561c223aadb3af298c2968e73ea39c6 /tmp/cvc5.zip" \| sha256sum --check --strict
	unzip -p /tmp/cvc5.zip '*/bin/cvc5' > /tmp/cvc5-bin
	sudo install -m 0755 /tmp/cvc5-bin /usr/local/bin/cvc5
	cvc5 --version \| head -1

	# Keyed on the download script and the shared pilot pin, so a change to the
	# pinned tag or the layout invalidates the cache. A restored cache is still
	# verified below.
	- name: Cache the OMG training corpus
	uses: actions/cache@v4
	with:
	path: examples/sysml-v2-training
	key: training-corpus-${{ hashFiles('scripts/download-training-examples.sh', 'scripts/pilot-pin.sh') }}

	- name: Download the OMG training corpus
	run: ./scripts/download-training-examples.sh

	- name: Verify the OMG training corpus
	run: \|
	count=$(find examples/sysml-v2-training -name '*.sysml' \| wc -l)
	echo "training corpus: $count .sysml files"
	if [ "$count" -eq 0 ]; then
	echo "error: the training corpus is empty; the corpus gate would not run" >&2
	exit 1
	fi

	# Same keying as the training corpus: the download script plus the shared
	# pilot pin. A restored cache is verified below rather than trusted.
	- name: Cache the OMG pilot corpora
	uses: actions/cache@v4
	with:
	path: examples/pilot-corpora
	key: pilot-corpora-${{ hashFiles('scripts/download-pilot-corpora.sh', 'scripts/pilot-pin.sh') }}

	- name: Download the OMG pilot corpora
	run: ./scripts/download-pilot-corpora.sh

	- name: Verify the OMG pilot corpora
	run: \|
	for root in sysml-examples sysml-validation kerml-examples; do
	count=$(find "examples/pilot-corpora/$root" -type f $ -name '.sysml' -o -name '.kerml' $ \| wc -l)
	echo "pilot corpus $root: $count model files"
	if [ "$count" -eq 0 ]; then
	echo "error: pilot corpus $root is empty; the corpora gate would not run" >&2
	exit 1
	fi
	done

	# Per-package timeout: under -race, passes and model run within 1% of go's 10m
	# default. Matches `make test`.
	- name: Run Go race tests
	run: make test

	- name: Upload coverage artifact
	uses: actions/upload-artifact@v4
	with:
	name: coverage
	path: coverage.txt
	if-no-files-found: error

	static-and-integrity:
	name: Go static and integrity checks
	needs: changes
	if: needs.changes.outputs.go == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 20
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod
	cache: true

	- name: Download Go modules
	run: go mod download

	- name: Install Z3
	run: \|
	sudo apt-get update
	sudo apt-get install -y z3
	z3 --version

	- name: Install cvc5
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/cvc5.zip \
	https://github.com/cvc5/cvc5/releases/download/cvc5-1.3.4/cvc5-Linux-x86_64-static.zip
	echo "dcdbfada0ce493ee98259c0816e0daafc561c223aadb3af298c2968e73ea39c6 /tmp/cvc5.zip" \| sha256sum --check --strict
	unzip -p /tmp/cvc5.zip '*/bin/cvc5' > /tmp/cvc5-bin
	sudo install -m 0755 /tmp/cvc5-bin /usr/local/bin/cvc5
	cvc5 --version \| head -1

	- name: Cache the OMG training corpus
	uses: actions/cache@v4
	with:
	path: examples/sysml-v2-training
	key: training-corpus-${{ hashFiles('scripts/download-training-examples.sh', 'scripts/pilot-pin.sh') }}

	- name: Download the OMG training corpus
	run: ./scripts/download-training-examples.sh

	- name: Verify the OMG training corpus
	run: \|
	count=$(find examples/sysml-v2-training -name '*.sysml' \| wc -l)
	echo "training corpus: $count .sysml files"
	if [ "$count" -eq 0 ]; then
	echo "error: the training corpus is empty; the corpus gate would not run" >&2
	exit 1
	fi

	- name: Cache the OMG pilot corpora
	uses: actions/cache@v4
	with:
	path: examples/pilot-corpora
	key: pilot-corpora-${{ hashFiles('scripts/download-pilot-corpora.sh', 'scripts/pilot-pin.sh') }}

	- name: Download the OMG pilot corpora
	run: ./scripts/download-pilot-corpora.sh

	- name: Verify the OMG pilot corpora
	run: \|
	for root in sysml-examples sysml-validation kerml-examples; do
	count=$(find "examples/pilot-corpora/$root" -type f $ -name '.sysml' -o -name '.kerml' $ \| wc -l)
	echo "pilot corpus $root: $count model files"
	if [ "$count" -eq 0 ]; then
	echo "error: pilot corpus $root is empty; the corpora gate would not run" >&2
	exit 1
	fi
	done

	- name: Check Go formatting
	run: \|
	unformatted=$(gofmt -l .)
	if [ -n "$unformatted" ]; then
	echo "The following files are not gofmt'd:"
	echo "$unformatted"
	exit 1
	fi

	- name: Run Go vet
	run: go vet ./...

	- name: Run static analysis (staticcheck + gosec)
	run: make lint

	# The embedded library snapshot is generated from the bundled OMG files;
	# a change to either without regenerating it must not land.
	- name: Check the library snapshot is current
	run: make stdlib-snapshot-check

	- name: Check grpc-go stays out of production code
	run: ./scripts/check-grpc-imports.sh

	- name: Lint the protobuf schema
	run: make proto-lint

	# Wire compatibility against the branch this pull request merges into;
	# the baseline is a git archive, so a shallow fetch of it is enough.
	- name: Check the protobuf schema for breaking changes
	env:
	BASE_REF: ${{ github.base_ref }}
	run: \|
	git fetch --no-tags --depth=1 origin "$BASE_REF"
	make proto-breaking BUF_BREAKING_REF="origin/$BASE_REF"

	# Re-run the corpus gate on its own so its verdict is legible in the log
	# and a skip is impossible to miss. TestCorpusGates is the shared
	# cache-independence case over all four OMG roots, which skips with them.
	- name: Run training corpus gate
	run: \|
	set -o pipefail
	go test -count=1 -v ./internal/core/model -run 'TestTrainingExamples\|TestCorpusGates' \| tee corpus-gate.log
	grep -E 'training files clean' corpus-gate.log
	if grep -qE '^\s*--- SKIP' corpus-gate.log; then
	echo "error: the corpus gate skipped" >&2
	exit 1
	fi

	# The pilot corpora gate on its own, for the same reason: its per-root
	# verdict lines are the reviewable output, and a skip must not pass.
	# It gates our own diagnostics only; the comparison against the reference
	# implementation (cmd/pilot-diff) needs Java validators and is not run here.
	- name: Run pilot corpora gate
	run: \|
	set -o pipefail
	go test -count=1 -v ./internal/core/model -run 'TestPilotCorpora' \| tee pilot-corpora-gate.log
	grep -E 'pilot corpus files clean' pilot-corpora-gate.log
	if grep -qE '^\s*--- SKIP' pilot-corpora-gate.log; then
	echo "error: the pilot corpora gate skipped" >&2
	exit 1
	fi

	# The RDF round-trip ratchet over every example on its own, so its
	# per-verdict counts are legible in the log and a skip must not pass.
	- name: Run RDF corpus round-trip gate
	run: \|
	set -o pipefail
	go test -count=1 -v ./internal/core/export -run 'TestCorpusRoundTrip' \| tee corpus-roundtrip-gate.log
	grep -E 'corpus round trip: [0-9]+ files:' corpus-roundtrip-gate.log
	if grep -qE '^\s*--- SKIP' corpus-roundtrip-gate.log; then
	echo "error: the RDF corpus round-trip gate skipped" >&2
	exit 1
	fi

	# Re-run the solver gate on its own, as the corpus gate is, so a skip
	# cannot hide behind a green suite.
	- name: Run solver gate (Z3)
	run: \|
	set -o pipefail
	go test -count=1 -v ./internal/core/solve ./internal/repl -run 'TestSolver\|TestDiscovery\|TestSolved\|TestDivisor\|TestCheck\|TestSolve\|TestConfigure\|TestSynthesis\|TestPinned\|TestFixed\|TestOptimize\|TestOptimum' \| tee solver-gate.log
	if grep -qE '^\s*--- SKIP' solver-gate.log; then
	echo "error: a solver-dependent test skipped" >&2
	exit 1
	fi

	# The differential gate compares the solver against the normative
	# evaluator; its summary counts make coverage drift reviewable in the log.
	- name: Run solver differential gate
	run: \|
	set -o pipefail
	go test -count=1 -v ./internal/core/solve -run TestDifferential \| tee differential-gate.log
	grep -E 'differential gate .: . elements:' differential-gate.log
	if grep -qE '^\s*--- SKIP' differential-gate.log; then
	echo "error: a differential gate skipped" >&2
	exit 1
	fi

	# Portability: the same subset run against both verified backends, so a
	# feature one of them refuses is reported rather than discovered by a user.
	# OPENSYSML_REQUIRE_SMT makes an absent solver a failure, not a skip.
	- name: Run solver portability gate
	run: \|
	set -o pipefail
	for solver in z3 cvc5; do
	OPENSYSML_SMT="$solver" go test -count=1 -v ./internal/core/solve \
	-run 'TestPortability' \| tee "portability-$solver.log"
	grep -E "portability of $solver" "portability-$solver.log"
	if grep -qE '^\s*--- SKIP' "portability-$solver.log"; then
	echo "error: the portability gate skipped for $solver" >&2
	exit 1
	fi
	done

	# The solver gate again against the second backend, not only the
	# portability subset: what z3 answers, cvc5 must answer too.
	- name: Run solver gate (cvc5)
	run: \|
	set -o pipefail
	OPENSYSML_SMT=cvc5 go test -count=1 -v ./internal/core/solve ./internal/repl \
	-run 'TestSolver\|TestDiscovery\|TestSolved\|TestDivisor\|TestCheck\|TestSolve\|TestConfigure\|TestSynthesis\|TestPinned\|TestFixed\|TestExplain\|TestCore' \
	\| tee cvc5-gate.log
	if grep -qE '^\s*--- SKIP' cvc5-gate.log; then
	echo "error: a solver-dependent test skipped under cvc5" >&2
	exit 1
	fi

	# The other half of the contract: with no solver to be found, every
	# solver-dependent test must skip with a reason and the absent-solver paths
	# must still report one. OPENSYSML_SMT names an executable that does not
	# exist, which is discovery's absent case, and the requirement flag is
	# dropped so a skip is the expected outcome here.
	- name: Run solver-absent gate
	run: \|
	set -o pipefail
	env -u OPENSYSML_REQUIRE_SMT OPENSYSML_SMT=/nonexistent/no-such-solver \
	go test -count=1 -v ./internal/core/solve ./internal/repl \| tee no-solver-gate.log
	for name in TestCheckReportsAnAbsentSolver TestExplainReportsAnAbsentSolver \
	TestSolveReportsAnAbsentSolver TestConfigureReportsAnAbsentSolver \
	TestOptimizeReportsAnAbsentSolver; do
	grep -qE "^\s*--- PASS: $name" no-solver-gate.log \|\| {
	echo "error: $name did not run without a solver" >&2
	exit 1
	}
	done

	# The client jobs download this binary rather than building one, so it runs
	# whenever the service or any client does.
	build:
	name: Build binaries
	needs: changes
	if: \|
	needs.changes.outputs.go == 'true' \|\|
	needs.changes.outputs.node == 'true' \|\|
	needs.changes.outputs.python == 'true' \|\|
	needs.changes.outputs.java == 'true' \|\|
	needs.changes.outputs.rust == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 10
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod
	cache: true

	- name: Download Go modules
	run: go mod download

	- name: Build binaries
	run: \|
	make build \
	VERSION="pr-${{ github.event.pull_request.number }}" \
	COMMIT="$(git rev-parse --short HEAD)" \
	BUILD_TIME="$(date -u '+%Y-%m-%d_%H:%M:%S')" \
	GO_VERSION="$(go version \| awk '{print $3}')"

	- name: Verify binaries
	run: \|
	./bin/sysml --version
	./bin/sysml-lsp --version
	ls -lh bin/

	- name: Upload binary artifacts
	uses: actions/upload-artifact@v4
	with:
	name: binaries
	path: bin/
	if-no-files-found: error

	# A gated job that did not run reports `skipped`, which this gate accepts; a job
	# skipped by a failed dependency is caught through that dependency's own result.
	build-and-test:
	name: Build and test
	if: always()
	needs:
	- changes
	- build
	- conformance-pkg
	- docs
	- java-test
	- node-test
	- python-test
	- race-tests
	- rust-test
	- static-and-integrity
	- vscode-extension
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 5
	steps:
	- name: Verify required checks
	env:
	CHANGES_RESULT: ${{ needs.changes.result }}
	BUILD_RESULT: ${{ needs.build.result }}
	CONFORMANCE_PKG_RESULT: ${{ needs.conformance-pkg.result }}
	DOCS_RESULT: ${{ needs.docs.result }}
	JAVA_TEST_RESULT: ${{ needs.java-test.result }}
	NODE_TEST_RESULT: ${{ needs.node-test.result }}
	PYTHON_TEST_RESULT: ${{ needs.python-test.result }}
	RACE_TESTS_RESULT: ${{ needs.race-tests.result }}
	RUST_TEST_RESULT: ${{ needs.rust-test.result }}
	STATIC_AND_INTEGRITY_RESULT: ${{ needs.static-and-integrity.result }}
	VSCODE_EXTENSION_RESULT: ${{ needs.vscode-extension.result }}
	run: \|
	ok() { [ "$1" = success ] \|\| [ "$1" = skipped ]; }
	if [ "$CHANGES_RESULT" != success ] \|\|
	! ok "$BUILD_RESULT" \|\|
	! ok "$CONFORMANCE_PKG_RESULT" \|\|
	! ok "$DOCS_RESULT" \|\|
	! ok "$JAVA_TEST_RESULT" \|\|
	! ok "$NODE_TEST_RESULT" \|\|
	! ok "$PYTHON_TEST_RESULT" \|\|
	! ok "$RACE_TESTS_RESULT" \|\|
	! ok "$RUST_TEST_RESULT" \|\|
	! ok "$STATIC_AND_INTEGRITY_RESULT" \|\|
	! ok "$VSCODE_EXTENSION_RESULT"; then
	echo "changes=$CHANGES_RESULT"
	echo "build=$BUILD_RESULT"
	echo "conformance-pkg=$CONFORMANCE_PKG_RESULT"
	echo "docs=$DOCS_RESULT"
	echo "java-test=$JAVA_TEST_RESULT"
	echo "node-test=$NODE_TEST_RESULT"
	echo "python-test=$PYTHON_TEST_RESULT"
	echo "race-tests=$RACE_TESTS_RESULT"
	echo "rust-test=$RUST_TEST_RESULT"
	echo "static-and-integrity=$STATIC_AND_INTEGRITY_RESULT"
	echo "vscode-extension=$VSCODE_EXTENSION_RESULT"
	exit 1
	fi

	vscode-extension:
	name: VS Code extension
	needs: changes
	if: needs.changes.outputs.vscode == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 10
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	- name: Set up Node.js
	uses: actions/setup-node@v4
	with:
	node-version: '22'
	cache: npm
	cache-dependency-path: editors/vscode/package-lock.json

	# Type-check, bundle and package. The generated TextMate grammars are
	# gated by go test ./editors/... in the job above.
	- name: Package the VS Code extension
	run: make vscode-package

	- name: Upload VS Code extension artifact
	uses: actions/upload-artifact@v4
	with:
	name: vscode-extension
	path: editors/vscode/opensysml-sysml.vsix
	if-no-files-found: error

	docs:
	name: Documentation site
	needs: changes
	if: needs.changes.outputs.docs == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 15
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	cache: pip
	cache-dependency-path: docs-requirements.txt

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: go.mod
	cache: true

	- name: Install documentation toolchain
	run: make docs-install

	- name: Check documentation links
	run: python3 scripts/check-doc-links.py

	# The shipped pages are generated from each command's description, so a
	# flag added without regenerating them would ship a stale page.
	- name: Verify the committed manual pages are current
	run: \|
	sudo apt-get update && sudo apt-get install -y --no-install-recommends mandoc
	make man-check

	# Internal work-item labels have no public referent, so they may not
	# appear in the pages a user reads.
	- name: Check for internal labels in reader-facing documentation
	run: python3 scripts/check-doc-ids.py

	# An oracle total quoted outside the generated block is a snapshot, and a
	# reader cannot tell that from a current figure unless the page says so.
	- name: Check quoted oracle figures name the round they measured
	run: python3 scripts/check-doc-figures.py

	# The census figures are derived from the committed baseline, and every
	# table row must be a constraint the baseline records (and vice versa).
	- name: Check the validation-constraint census matches its baseline
	run: go run ./cmd/validation-census -check

	# Changelog entries live in changes/unreleased/ until a release folds them in.
	- name: Check changelog fragments
	run: python3 scripts/changelog-test.py && python3 scripts/changelog.py check

	# The compliance census is counted when the site is built, never committed.
	- name: Check the compliance census hook
	run: python3 scripts/mkdocs_census-test.py

	# --strict: a link or anchor a moved page left behind fails here rather than
	# publishing a 404 from main.
	- name: Build documentation site
	run: make docs

	node-test:
	name: Node client tests
	needs: [changes, build]
	if: needs.changes.outputs.node == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@v4

	# The client verifies a download against the copy of the pinned digests it
	# ships, so a copy that drifts from the table pins something else.
	- name: Check the clients ship the pinned release digests
	run: python3 scripts/sync-release-digests.py --check

	- uses: actions/download-artifact@v4
	with:
	name: binaries
	path: bin

	- uses: actions/setup-node@v4
	with:
	node-version: '22'
	cache: npm
	cache-dependency-path: clients/node/package-lock.json

	# The tests and the conformance runner start private children of this
	# binary; without it they would build one from the checkout, needing Go.
	- name: Install the sysml-grpc binary
	run: \|
	chmod +x bin/sysml-grpc
	echo "OPENSYSML_BINARY=$(pwd)/bin/sysml-grpc" >> "$GITHUB_ENV"

	- name: Install the client's dependencies
	working-directory: clients/node
	# --ignore-scripts: no dependency of this client needs an install hook,
	# and one added upstream would run here with the workflow's token.
	run: npm ci --ignore-scripts

	- name: Build, typecheck and lint
	working-directory: clients/node
	run: \|
	npm run build
	npm run typecheck
	npm run lint

	- name: Run the tests
	working-directory: clients/node
	run: npm test

	# --allow-skips because v1 covers 5 of the 15 RPCs; the skipped count is
	# reported and the artifact records which scenarios they were.
	- name: Run the conformance suite through the client
	working-directory: clients/node
	run: \|
	npm run conformance -- --binary "$OPENSYSML_BINARY" --allow-skips \
	--report node-conformance.json

	# A runner that passes against a deliberately broken client tests nothing.
	- name: Check the conformance runner catches a broken client
	working-directory: clients/node
	run: \|
	for mutation in hide-capability drop-diagnostics blank-symbol-kind shift-integer drop-feature-values; do
	if npm run conformance -- --binary "$OPENSYSML_BINARY" --allow-skips \
	--protocols connect --mutate "$mutation" > /dev/null 2>&1; then
	echo "error: the suite passed with the $mutation mutation applied" >&2
	exit 1
	fi
	echo "ok: $mutation is caught"
	done

	- uses: actions/upload-artifact@v4
	with:
	name: node-conformance
	path: clients/node/node-conformance.json

	# Pinned by digest, as cvc5 is above, so a substituted asset is not run.
	- name: Install buf
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/buf \
	https://github.com/bufbuild/buf/releases/download/v1.57.2/buf-Linux-x86_64
	echo "644e6fa4a62afc2b0d425d3ac9538d2f512947c0f2e82fc95c41a5eab6729414 /tmp/buf" \| sha256sum --check --strict
	sudo install -m 0755 /tmp/buf /usr/local/bin/buf
	buf --version

	# Committed stubs must be what buf generates, or the checked-in client and
	# the schema can drift apart silently.
	- name: Verify the committed TypeScript stubs are current
	run: \|
	make proto-ts BUF=buf
	git add -N clients/node/src/generated
	git diff --exit-code clients/node/src/generated

	python-test:
	name: Python client tests
	needs: [changes, build]
	if: needs.changes.outputs.python == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 10
	steps:
	- name: Check out repository
	uses: actions/checkout@v4

	# The client verifies a download against the copy of the pinned digests it
	# ships, so a copy that drifts from the table pins something else.
	- name: Check the clients ship the pinned release digests
	run: python3 scripts/sync-release-digests.py --check

	- name: Download binary artifacts
	uses: actions/download-artifact@v4
	with:
	name: binaries
	path: bin

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'

	- name: Install sysml-grpc
	run: \|
	mkdir -p ~/.opensysml/bin
	cp bin/sysml-grpc ~/.opensysml/bin/
	chmod +x ~/.opensysml/bin/sysml-grpc

	- name: Install Python client
	run: \|
	make python-install
	# Pinned and wheel-only: an unpinned resolve runs whatever was
	# published today, and a source distribution runs its own build code.
	pip install --only-binary :all: pytest==9.0.3 pytest-mock==3.15.1 psutil==7.2.2

	# The integration tests connect to a service on the standard port with
	# auto_start=False, the explicit opt-in to one the client does not manage;
	# without it they skip, exercising nothing, so one runs here for the rest of
	# the job. Tests naming no address start a private child of their own.
	- name: Start sysml-grpc
	run: nohup ~/.opensysml/bin/sysml-grpc -port 50051 > /tmp/sysml-grpc.log 2>&1 &

	- name: Wait for sysml-grpc
	run: \|
	python3 - <<'PY'
	import socket, sys, time

	for _ in range(30):
	with socket.socket() as probe:
	probe.settimeout(1)
	if probe.connect_ex(("localhost", 50051)) == 0:
	print("sysml-grpc is listening on 50051")
	sys.exit(0)
	time.sleep(1)
	sys.exit("sysml-grpc never listened on 50051")
	PY

	# OPENSYSML_REQUIRE_SERVICE turns "no service, so skip" into a failure: a
	# service is provided here, so its absence is the bug, not a pass.
	- name: Run Python client tests
	env:
	OPENSYSML_REQUIRE_SERVICE: 1
	run: make python-test

	- name: Verify Python client import
	run: python3 -c "import opensysml; print(f'opensysml {opensysml.__version__} imported successfully')"

	# Pinned by digest, so a substituted asset is not run.
	- name: Install buf
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/buf \
	https://github.com/bufbuild/buf/releases/download/v1.57.2/buf-Linux-x86_64
	echo "644e6fa4a62afc2b0d425d3ac9538d2f512947c0f2e82fc95c41a5eab6729414 /tmp/buf" \| sha256sum --check --strict
	sudo install -m 0755 /tmp/buf /usr/local/bin/buf
	buf --version

	# Committed stubs must be what buf generates, or the checked-in client and
	# the schema can drift apart silently.
	- name: Verify the committed Python stubs are current
	run: \|
	# Pinned to the version recorded in the committed stubs
	# (GRPC_GENERATED_VERSION), so a newer release cannot fail this check.
	pip install --only-binary :all: grpcio-tools==1.83.0
	make python-proto BUF=buf
	git add -N clients/python/opensysml/proto
	git diff --exit-code clients/python/opensysml/proto

	rust-test:
	name: Rust client tests
	needs: [changes, build]
	if: needs.changes.outputs.rust == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@v4

	# The client verifies a download against the copy of the pinned digests it
	# ships, so a copy that drifts from the table pins something else.
	- name: Check the clients ship the pinned release digests
	run: python3 scripts/sync-release-digests.py --check

	# Reuse the service built by the Go job rather than installing Go again
	# in this Rust-focused gate.
	- uses: actions/download-artifact@v4
	with:
	name: binaries
	path: bin

	# The artifact is a zip, which does not carry the executable bit.
	- name: Install the sysml-grpc binary
	run: chmod +x bin/sysml-grpc

	- name: Install Rust 1.83
	run: \|
	rustup toolchain install 1.83.0 --profile minimal
	rustup default 1.83.0
	rustup component add rustfmt clippy --toolchain 1.83.0
	rustc --version
	cargo --version

	- name: Check Rust formatting
	working-directory: clients/rust
	run: cargo fmt --all -- --check

	- name: Run Rust clippy
	working-directory: clients/rust
	run: cargo clippy --all-targets -- -D warnings

	# A missing service must fail rather than silently skipping integration
	# tests, because the downloaded binary is required by this job.
	- name: Run Rust tests
	working-directory: clients/rust
	env:
	OPENSYSML_GRPC_BINARY: ${{ github.workspace }}/bin/sysml-grpc
	OPENSYSML_REQUIRE_SERVICE: 1
	run: cargo test --workspace

	# Invoke the runner directly so this job does not rebuild the service
	# through make conformance-rust, which would require Go here.
	- name: Run Rust conformance
	env:
	OPENSYSML_GRPC_BINARY: ${{ github.workspace }}/bin/sysml-grpc
	run: \|
	cargo run --manifest-path clients/rust/Cargo.toml -p opensysml-conformance -- \
	-binary "$GITHUB_WORKSPACE/bin/sysml-grpc" \
	-report bin/conformance-report-rust.json

	- name: Package Rust client
	working-directory: clients/rust
	run: cargo package -p opensysml

	# The digest pins the generator binary so a substituted release cannot
	# change the committed Rust artifacts.
	- name: Install buf
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/buf \
	https://github.com/bufbuild/buf/releases/download/v1.57.2/buf-Linux-x86_64
	echo "644e6fa4a62afc2b0d425d3ac9538d2f512947c0f2e82fc95c41a5eab6729414 /tmp/buf" \| sha256sum --check --strict
	chmod +x /tmp/buf
	/tmp/buf --version

	- name: Verify Rust generated artifacts
	run: \|
	make BUF=/tmp/buf proto-rust
	git add -N clients/rust/opensysml/src/proto clients/rust/conformance/sysml.descriptor.binpb
	git diff --exit-code clients/rust/opensysml/src/proto clients/rust/conformance/sysml.descriptor.binpb

	- uses: actions/upload-artifact@v4
	with:
	name: conformance-report-rust
	path: bin/conformance-report-rust.json

	# Mirrors the CircleCI java-test job: unit tests plus the conformance suite over
	# both Connect encodings, on the client's Java 17 baseline.
	java-test:
	name: Java client tests
	needs: [changes, build]
	if: needs.changes.outputs.java == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v4

	# The client verifies a download against the copy of the pinned digests it
	# ships, so a copy that drifts from the table pins something else.
	- name: Check the clients ship the pinned release digests
	run: python3 scripts/sync-release-digests.py --check

	- uses: actions/download-artifact@v4
	with:
	name: binaries
	path: bin

	# 17 is the client's compile target, so a 21-only construct fails here.
	- uses: actions/setup-java@v4
	with:
	distribution: temurin
	java-version: '17'
	cache: maven

	- name: Make the service binary executable
	run: chmod +x bin/sysml-grpc

	# opensysml.requireService turns "no binary, so skip" into a failure, and
	# `install` builds the sources and javadoc jars a release needs on every commit.
	- name: Test and install the client
	run: mvn -B -f clients/java/pom.xml install -Dopensysml.requireService=true

	- name: Run the conformance suite through the client
	run: \|
	mvn -B -q -f clients/java/pom.xml -pl opensysml-conformance \
	dependency:build-classpath -Dmdep.outputFile=/tmp/cp.txt
	java -cp "clients/java/opensysml-conformance/target/classes:$(cat /tmp/cp.txt)" \
	org.openmbee.opensysml.conformance.Main \
	-binary bin/sysml-grpc -protocols connect,connect-json -allow-skips \
	-report conformance-report-java.json

	- uses: actions/upload-artifact@v4
	if: always()
	with:
	name: java-conformance-report
	path: conformance-report-java.json

	# The Java message classes come out of the one buf template that also runs the
	# Go plugins through `go run`, so the check needs a Go toolchain even here.
	- uses: actions/setup-go@v5
	with:
	go-version-file: go.mod
	cache: true

	# Pinned by digest, so a substituted asset is not run.
	- name: Install buf
	run: \|
	curl -fsSL --proto '=https' --proto-redir '=https' -o /tmp/buf \
	https://github.com/bufbuild/buf/releases/download/v1.57.2/buf-Linux-x86_64
	echo "644e6fa4a62afc2b0d425d3ac9538d2f512947c0f2e82fc95c41a5eab6729414 /tmp/buf" \| sha256sum --check --strict
	sudo install -m 0755 /tmp/buf /usr/local/bin/buf
	buf --version

	# Committed stubs must be what buf generates, or the checked-in client and
	# the schema can drift apart silently.
	- name: Verify the committed Java stubs are current
	run: \|
	make proto-buf BUF=buf
	git add -N clients/java/opensysml-client/src/main/java/org/openmbee/opensysml/proto
	git diff --exit-code clients/java/opensysml-client/src/main/java/org/openmbee/opensysml/proto

	conformance-pkg:
	name: Conformance suite
	needs: changes
	if: needs.changes.outputs.go == 'true'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@v4

	- uses: actions/setup-go@v5
	with:
	go-version-file: go.mod
	cache: true

	- name: Download modules
	run: go mod download

	# The runner builds and starts sysml-grpc itself, so this gate proves the
	# whole path a client takes, not a pre-started service.
	- name: Run conformance suite
	run: make conformance

	# The same suite against the legacy grpc-go transport, so -transport grpc
	# cannot rot untested.
	- name: Run conformance suite against -transport grpc
	run: go run ./cmd/conformance -transport grpc -protocols grpc -report bin/conformance-grpc-transport-report.json

	# The same suite through the public Go API (client/opensysml); the pkg
	# protocol runs in-process and pkg-connect dials the service, so this proves
	# the API answers as the wire does. Scenarios outside its v1 surface are
	# counted skips, not hidden.
	- name: Conformance suite through client/opensysml
	run: make conformance-pkg

	# if: always() — the reports matter most when a gate failed and halted the job.
	- uses: actions/upload-artifact@v4
	if: always()
	with:
	name: conformance-reports
	path: \|
	bin/conformance-report.json
	bin/conformance-report.xml
	bin/conformance-grpc-transport-report.json
	bin/conformance-pkg-report.json

Navigation Menu

Uh oh!

feat(runtime): report execution choice points #252

Workflow file

feat(runtime): report execution choice points #252

Uh oh!

Workflow file for this run