diff --git a/.github/WORKFLOWS_SUMMARY.md b/.github/WORKFLOWS_SUMMARY.md index 2f60858b..73d0ed8a 100644 --- a/.github/WORKFLOWS_SUMMARY.md +++ b/.github/WORKFLOWS_SUMMARY.md @@ -2,6 +2,14 @@ ## โœ… **FINAL STATE - Production-Ready CI/CD** +### ๐Ÿงช **CI Matrix Build Status** + +| Module | Status | +|--------|--------| +| core | ![core](https://github.com/flowforge/flowforge/actions/workflows/ci.yml/badge.svg?job=core) | +| connectors | ![connectors](https://github.com/flowforge/flowforge/actions/workflows/ci.yml/badge.svg?job=connectors) | +| engines | ![engines](https://github.com/flowforge/flowforge/actions/workflows/ci.yml/badge.svg?job=engines) | + ### ๐Ÿš€ **Active Workflows (8 files)** | Workflow | Purpose | Status | Triggers | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ade71c2..979f2c22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,32 @@ env: SBT_OPTS: -Xmx4G -XX:+UseG1GC jobs: - build-and-test: - name: Build and test + build: + strategy: + matrix: + include: + - module: core + commands: > + scalafmtCheckAll scalafixAll core/test compile-fail-tests/test + paths: | + target + modules/core/target + modules/compile-fail-tests/target + - module: connectors + commands: > + connectors/test connectors-gcs/test + paths: | + target + modules/connectors/target + modules/connectors-gcs/target + - module: engines + commands: > + engines-spark/test engines-flink/test "engines-spark/testOnly *StreamingCDCSpec" "engines-flink/testOnly *EngineAbstractionSpec" + paths: | + target + modules/engines-spark/target + modules/engines-flink/target + name: ${{ matrix.module }} runs-on: ubuntu-22.04 steps: - name: Checkout @@ -31,58 +55,13 @@ jobs: ~/.ivy2/cache ~/.cache/coursier ~/.sbt - key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt', '**/project/**/*.sbt', '**/project/**/Dependencies.scala') }} + ${{ matrix.paths }} + key: ${{ runner.os }}-sbt-${{ matrix.module }}-${{ hashFiles('**/build.sbt', '**/project/**/*.sbt', format('modules/{0}/**', matrix.module)) }} restore-keys: | - ${{ runner.os }}-sbt- - - - name: Check formatting and linting - run: sbt scalafmtCheckAll scalafixAll - - - name: Compile all modules - run: sbt compile Test/compile - - - name: Run tests with coverage - run: | - sbt coverage test - sbt coverageReport - sbt coverageAggregate - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - with: - file: target/scala-2.13/scoverage-report/scoverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false + ${{ runner.os }}-sbt-${{ matrix.module }}- - - name: Run compile-fail tests - run: | - echo "๐Ÿ”ฅ Running compile-fail tests - proving FlowForge's core USP" - sbt "compile-fail-tests / test" - echo "โœ… Compile-fail tests passed - contract drift prevention verified!" - - - name: Spark smoke test - run: | - echo "๐Ÿš€ Running Spark local smoke test" - sbt "engines-spark/testOnly *StreamingCDCSpec" - echo "โœ… Spark smoke test passed!" - - - name: Flink smoke test - run: | - echo "๐ŸŒŠ Running Flink smoke test" - sbt "engines-flink/testOnly *EngineAbstractionSpec" - echo "โœ… Flink smoke test passed!" - - - name: Test g8 template generation - run: | - echo "๐Ÿ“‹ Testing g8 template generation" - cd /tmp - sbt new file://${{ github.workspace }}/flowforge.g8 --name="ci-test-pipeline" --organization="com.flowforge.ci" - cd ci-test-pipeline - echo "๐Ÿ”จ Testing generated project compilation" - sbt compile - echo "๐Ÿงช Testing generated project tests" - sbt test - echo "โœ… G8 template test passed - turnkey experience verified!" + - name: Build module + run: sbt -batch ${{ matrix.commands }} spark-it: if: github.event_name == 'workflow_dispatch' @@ -106,4 +85,4 @@ jobs: ${{ runner.os }}-sbt- - name: Run opt-in Spark/Delta ITs run: | - sbt -DwithSparkIT=true "enginesSpark/testOnly *SparkDeltaSCD2IT" + sbt -batch -DwithSparkIT=true "enginesSpark/testOnly *SparkDeltaSCD2IT"