v1.0-RC Readiness #106

Workflow file for this run

	name: CI

	on:
	push:
	branches: [ main ]
	paths:
	- 'modules/**'
	- 'build.sbt'
	- 'project/**'
	- '.github/workflows/ci.yml'
	- '!docs/**'
	- '!*/.md'
	pull_request:
	branches: [ main ]
	paths:
	- 'modules/**'
	- 'build.sbt'
	- 'project/**'
	- '.github/workflows/ci.yml'
	workflow_dispatch:

	permissions:
	contents: read

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	JAVA_OPTS: -Xmx4G -XX:+UseG1GC
	SBT_OPTS: -Xmx4G -XX:+UseG1GC

	jobs:
	build:
	strategy:
	fail-fast: false
	matrix:
	include:
	- module: core
	commands: >
	scalafmtCheckAll "scalafixAll --check" core/test compile-fail-tests/test
	paths: \|
	target
	modules/core/target
	modules/compile-fail-tests/target
	- module: connectors
	commands: >
	scalafmtCheckAll "scalafixAll --check" connectors/test connectors-gcs/test
	paths: \|
	target
	modules/connectors/target
	modules/connectors-gcs/target
	- module: engines
	commands: >
	scalafmtCheckAll "scalafixAll --check" engines-spark/test engines-flink/test
	paths: \|
	target
	modules/engines-spark/target
	modules/engines-flink/target
	name: ${{ matrix.module }}
	runs-on: ubuntu-22.04
	if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
	steps:
	- name: SBT (build matrix)
	uses: ./.github/actions/sbt
	with:
	java-version: '17'
	distribution: temurin
	commands: ${{ matrix.commands }}

	refactor-lint:
	name: Refactor lint & Unsafe cast gate
	runs-on: ubuntu-22.04
	if: ${{ !github.event.pull_request.draft }}
	steps:
	- name: Scalafmt check
	uses: ./.github/actions/sbt
	with:
	java-version: '17'
	distribution: temurin
	commands: scalafmtCheckAll
	- name: Scalafix check (production code only - test/examples excluded)
	run: \|
	# Run scalafix on production sources (excluding tests, examples, CLIs)
	# Examples are demo code, test code violations are documented technical debt
	echo "🔍 Running scalafix on production code (excluding examples & CLIs)..."

	# Run scalafix on core production code (examples excluded by directory pattern)
	# The compile scope already excludes test sources
	sbt -batch \
	"core/compile:scalafix --check" \
	"connectors/compile:scalafix --check" \
	"connectors-gcs/compile:scalafix --check" \
	"infrastructure/compile:scalafix --check" \
	"engines-spark/compile:scalafix --check" \
	"engines-flink/compile:scalafix --check" \
	"quality-deequ/compile:scalafix --check" \
	2>&1 \| tee scalafix-output.txt

	# Check for actual errors (not warnings) excluding examples directory
	if grep "error: \[" scalafix-output.txt \| grep -v "examples/" \| grep -q "\.scala:"; then
	echo "❌ Scalafix violations found in production code"
	echo "Violations:"
	grep "error: \[" scalafix-output.txt \| grep -v "examples/" \| grep "\.scala:" \| head -n 20
	exit 1
	fi

	echo "✅ Scalafix quality gates passed for production code"

	doc-lint:
	name: Scaladoc coverage (report → enforce later)
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Install ripgrep
	run: sudo apt-get update && sudo apt-get install -y ripgrep
	- name: Scan public packages for missing Scaladoc
	shell: bash
	run: \|
	set -euo pipefail
	ROOTS=(
	modules/core/src/main/scala/com/flowforge/core
	modules/core/src/main/scala/com/flowforge/framework
	modules/core/src/main/scala/com/flowforge/core/contracts
	modules/core/src/main/scala/com/flowforge/core/algebra
	modules/engines-spark/src/main/scala/com/flowforge/engines/spark
	modules/quality-deequ/src/main/scala/com/flowforge/quality/deequ
	)
	total=0; missing=0
	mapfile -t FILES < <(rg -l "^" "${ROOTS[@]}" --glob "*/.scala" \|\| true)
	offenders=()
	for file in "${FILES[@]}"; do
	# Classes/traits/objects
	while IFS= read -r line; do
	ln=${line%%:}; decl=${line#:*:}
	(( total++ ))
	start=$((ln-3)); [ $start -lt 1 ] && start=1
	if ! sed -n "$start,$((ln-1))p" "$file" \| rg -q "^/\\"; then
	offenders+=("$file:$ln:$decl")
	(( missing++ ))
	fi
	done < <(rg -n "^\s*(final\s+)?(case\s+)?(class\|trait\|object)\s+\w+" "$file" \|\| true)
	# Public defs (basic heuristic)
	while IFS= read -r line; do
	ln=${line%%:}; decl=${line#:*:}
	(( total++ ))
	start=$((ln-3)); [ $start -lt 1 ] && start=1
	if ! sed -n "$start,$((ln-1))p" "$file" \| rg -q "^/\\"; then
	offenders+=("$file:$ln:$decl")
	(( missing++ ))
	fi
	done < <(rg -n "^\sdef\s+\w+\s\(" "$file" \| rg -v "^\s*(private\|protected)\b" \|\| true)
	done
	cov=$(( ( (total-missing) * 100 ) / ( (total==0) ? 1 : total ) ))
	echo "Scaladoc coverage: $cov% ($((total-missing))/$total)"
	if [ ${#offenders[@]} -gt 0 ]; then
	echo "First 200 missing docs:"; printf '%s\n' "${offenders[@]}" \| head -n 200
	fi
	# ENFORCE: fail when offenders exist (single CI run post-sweep)
	if [ ${#offenders[@]} -gt 0 ]; then
	echo "❌ Missing Scaladoc found"; exit 1
	fi

	- name: Whitelist checks (asInstanceOf, scala.util.Try)
	shell: bash
	run: \|
	echo "🔍 Enforcing whitelists (scalafix doesn't support per-file excludes)"
	set -euo pipefail
	MAPFILE=()
	while IFS= read -r -d '' f; do MAPFILE+=("$f"); done < <(find modules -type f -path "/src/main/scala/" -name "*.scala" -print0)

	# asInstanceOf whitelist
	ASINSTANCEOF_WHITELIST=(
	"modules/engines-spark/src/main/scala/com/flowforge/engines/spark/SparkDataAlgebra.scala"
	"modules/core/src/main/scala/com/flowforge/core/contracts/internal/ContractMacros.scala"
	)

	# scala.util.Try whitelist
	TRY_WHITELIST=(
	"modules/core/src/main/scala/com/flowforge/core/safety/Safety.scala"
	"modules/core/src/main/scala/com/flowforge/core/package.scala"
	)

	is_asinstanceof_whitelisted() {
	local f="$1"
	for w in "${ASINSTANCEOF_WHITELIST[@]}"; do [[ "$f" == "$w" ]] && return 0; done
	return 1
	}

	is_try_whitelisted() {
	local f="$1"
	for w in "${TRY_WHITELIST[@]}"; do [[ "$f" == "$w" ]] && return 0; done
	return 1
	}

	FAIL=0

	# Check asInstanceOf
	while IFS= read -r line; do
	file=$(echo "$line" \| awk -F: '{print $1}')
	if is_asinstanceof_whitelisted "$file"; then continue; fi
	echo "❌ asInstanceOf found in $file (use pattern matching instead)"
	FAIL=1
	done < <(rg -n "\\basInstanceOf\\b" "${MAPFILE[@]}" \|\| true)

	# Check scala.util.Try
	while IFS= read -r line; do
	file=$(echo "$line" \| awk -F: '{print $1}')
	if is_try_whitelisted "$file"; then continue; fi
	echo "❌ scala.util.Try found in $file (use Safety.safely instead)"
	FAIL=1
	done < <(rg -n "\\bscala\\.util\\.Try\\b" "${MAPFILE[@]}" \|\| true)

	if [[ "$FAIL" -ne 0 ]]; then exit 1; fi
	echo "✅ PASS: Whitelists enforced (asInstanceOf, scala.util.Try)"


	spark-it:
	name: Spark Integration Tests
	runs-on: ubuntu-22.04
	if: \|
	github.event_name == 'workflow_dispatch' \|\|
	(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-integration-tests')) \|\|
	(github.event_name == 'push' && github.ref == 'refs/heads/main')
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK 17
	uses: actions/setup-java@v4
	with:
	distribution: temurin
	java-version: '17'
	- name: Cache Ivy and Coursier
	uses: actions/cache@v4
	with:
	path: \|
	~/.ivy2/cache
	~/.cache/coursier
	~/.sbt
	key: ${{ runner.os }}-sbt-${{ hashFiles('/build.sbt', '/project/*/.sbt', '/project//Dependencies.scala') }}
	restore-keys: \|
	${{ runner.os }}-sbt-
	- name: Run Spark/Delta Integration Tests
	run: \|
	sbt -batch -DwithSparkIT=true "enginesSpark/testOnly *SparkDeltaSCD2IT"

Navigation Menu

v1.0-RC Readiness #106

Workflow file

v1.0-RC Readiness #106

Uh oh!

Workflow file for this run