CI #119
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - 'modules/**' | |
| - 'build.sbt' | |
| - 'project/**' | |
| - '.github/workflows/ci.yml' | |
| - '.github/actions/**' | |
| # Only run on main for coverage artifact generation (triggered by merge) | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - 'modules/**' | |
| - 'build.sbt' | |
| - 'project/**' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_OPTS: -Xmx4G -XX:+UseG1GC | |
| SBT_OPTS: -Xmx4G -XX:+UseG1GC | |
| jobs: | |
| quality: | |
| name: Code Quality | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Cache Coursier | |
| uses: coursier/cache-action@v6 | |
| - name: Scalafmt check | |
| run: sbt -batch scalafmtCheckAll | |
| - name: Scalafix check | |
| continue-on-error: true | |
| run: | | |
| sbt -batch \ | |
| "core/compile:scalafix --check" \ | |
| "connectors/compile:scalafix --check" \ | |
| "connectors-gcs/compile:scalafix --check" \ | |
| "infrastructure/compile:scalafix --check" \ | |
| "engines-spark/compile:scalafix --check" \ | |
| "engines-flink/compile:scalafix --check" \ | |
| "quality-deequ/compile:scalafix --check" | |
| build: | |
| needs: [quality] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - module: core | |
| commands: > | |
| scalafmtCheckAll core/test compile-fail-tests/test | |
| paths: | | |
| target | |
| modules/core/target | |
| modules/compile-fail-tests/target | |
| - module: connectors | |
| commands: > | |
| scalafmtCheckAll connectors/test connectors-gcs/test | |
| paths: | | |
| target | |
| modules/connectors/target | |
| modules/connectors-gcs/target | |
| - module: engines | |
| commands: > | |
| scalafmtCheckAll engines-spark/test engines-flink/test | |
| paths: | | |
| target | |
| modules/engines-spark/target | |
| modules/engines-flink/target | |
| - module: jdbc-quality | |
| commands: > | |
| scalafmtCheckAll connectors-jdbc/test quality-deequ/test | |
| paths: | | |
| target | |
| modules/connectors-jdbc/target | |
| modules/quality-deequ/target | |
| - module: clis | |
| commands: > | |
| scalafmtCheckAll validation-cli/test contracts-extractor-cli/test maintenance-cli/test | |
| paths: | | |
| target | |
| modules/validation-cli/target | |
| modules/contracts-extractor-cli/target | |
| modules/maintenance-cli/target | |
| - module: sdk-experimental | |
| commands: > | |
| scalafmtCheckAll contracts-sdk/test experimental/test | |
| paths: | | |
| target | |
| modules/contracts-sdk/target | |
| modules/experimental/target | |
| name: ${{ matrix.module }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: SBT (build matrix with coverage) | |
| uses: ./.github/actions/sbt | |
| with: | |
| java-version: '17' | |
| distribution: temurin | |
| commands: coverage ${{ matrix.commands }} ; coverageReport | |
| - name: Upload coverage data | |
| if: always() && github.ref == 'refs/heads/main' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-${{ matrix.module }} | |
| path: | | |
| target/scala-*/scoverage-data | |
| modules/*/target/scala-*/scoverage-data | |
| modules/*/target/scala-*/scoverage-report | |
| retention-days: 1 | |
| if-no-files-found: warn | |
| security: | |
| name: Security Scans | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: CodeQL Analysis | |
| uses: github/codeql-action/init@v3 | |
| with: | |
| languages: 'java' # CodeQL uses 'java' for Scala (analyzes JVM bytecode) | |
| - name: Autobuild | |
| uses: github/codeql-action/autobuild@v3 | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v3 | |
| - name: Secret Scan | |
| uses: trufflesecurity/trufflehog@main | |
| with: | |
| path: ./ | |
| base: ${{ github.event.pull_request.base.sha }} | |
| head: HEAD | |
| extra_args: --only-verified=false | |
| - name: Semgrep SAST | |
| uses: returntocorp/semgrep-action@v1 | |
| with: | |
| config: auto | |
| spark-it: | |
| needs: [build] | |
| name: Spark Integration Tests | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-integration-tests')) || | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| - name: Cache Ivy and Coursier | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.cache/coursier | |
| ~/.sbt | |
| key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt', '**/project/**/*.sbt', '**/project/**/Dependencies.scala') }} | |
| restore-keys: | | |
| ${{ runner.os }}-sbt- | |
| - name: Run Spark/Delta Integration Tests | |
| uses: nick-fields/retry-action@v3 | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| retry_on: error | |
| command: sbt -batch -DwithSparkIT=true "engines-spark/testOnly *SparkDeltaSCD2IT" |