java-pr.yml

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Checks that are intended to run on PRs containing Java code.

name: Java PR

on:
  pull_request:
    branches:
      - 'main'
    paths:
      - '**.java'
      - '**.xml'
      # Include Python requirements files for dependency updates (e.g., for v2 templates).
      - '**/requirements.txt'
      - 'plugins/core-plugin/src/main/resources/**'
      # Include relevant GitHub Action files for running these checks.
      # This will make it easier to verify action changes don't break anything.
      - '.github/actions/setup-env/*'
      - '.github/workflows/java-pr.yml'
      # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml)
      - '!v2/datastream-to-spanner/**'
      - '!v2/spanner-common/**'
      - '!v2/spanner-change-streams-to-sharded-file-sink/**'
      - '!v2/gcs-to-sourcedb/**'
      - '!v2/spanner-migrations-sdk/**'
      - '!v2/spanner-custom-shard/**'
      - '!v2/sourcedb-to-spanner/**'
      - '!v2/spanner-to-sourcedb/**'
      - '!v2/gcs-spanner-dv/**'
      # Exclude kafka paths from global run (covered in kafka-pr.yml)
      - '!v2/kafka-to-bigquery/**'
      - '!v2/kafka-to-gcs/**'
      - '!v2/kafka-to-kafka/**'
      - '!v2/pubsub-to-kafka/**'
      # Exclude yaml and python paths from global run (covered in yaml-pr.yml)
      - '!yaml/**'
      - '!python/**'
  schedule:
    - cron: "0 */12 * * *"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
  IT_REGION: us-west2

permissions:
  contents: read

jobs:
  spotless_check:
    name: Spotless
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Setup Environment
        id: setup-env
        uses: ./.github/actions/setup-env
      - name: Run Spotless
        run: ./cicd/run-spotless
  checkstyle_check:
    name: Checkstyle
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Setup Environment
        id: setup-env
        uses: ./.github/actions/setup-env
      - name: Run Checkstyle
        run: ./cicd/run-checkstyle
  java_build:
    name: Build
    timeout-minutes: 60
    runs-on: [self-hosted, it]
    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Setup Environment
        id: setup-env
        uses: ./.github/actions/setup-env
      - name: Run Build
        run: ./cicd/run-build
      - name: Cleanup Java Environment
        uses: ./.github/actions/cleanup-java-env
  java_unit_tests:
    name: Unit Tests
    needs: [java_build]
    timeout-minutes: 60
    runs-on: [self-hosted, it]
    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Setup Environment
        id: setup-env
        uses: ./.github/actions/setup-env
      - name: Run Unit Tests
        run: ./cicd/run-unit-tests
      - name: Upload Unit Tests Report
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always() # always run even if the previous step fails
        with:
          name: surefire-unit-test-results
          path: |
            **/surefire-reports/TEST-*.xml
            **/surefire-reports/*.html
            **/surefire-reports/html/**
          retention-days: 1
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: GoogleCloudPlatform/DataflowTemplates
          files: 'target/site/jacoco-aggregate/jacoco.xml'
      - name: Cleanup Java Environment
        uses: ./.github/actions/cleanup-java-env
        if: always()
  java_integration_smoke_tests_templates:
    name: Dataflow Templates Integration Smoke Tests
    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
    timeout-minutes: 60
    # Run on any runner that matches all the specified runs-on values.
    runs-on: [self-hosted, it]
    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Setup Environment
        id: setup-env
        uses: ./.github/actions/setup-env
      - name: Run Integration Smoke Tests
        run: |
          ./cicd/run-it-smoke-tests \
          --modules-to-build="DEFAULT" \
          --it-region="${{ env.IT_REGION }}" \
          --it-project="cloud-teleport-testing" \
          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
          --it-private-connectivity="datastream-connect-2" \
          --it-base-container-image=":dev-image-latest" \
          --it-base-python-container-image=":dev-image-latest"
      - name: Upload Smoke Tests Report
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always() # always run even if the previous step fails
        with:
          name: surefire-smoke-test-results
          path: |
            **/surefire-reports/TEST-*.xml
            **/surefire-reports/*.html
            **/surefire-reports/html/**
          retention-days: 1
      - name: Cleanup Java Environment
        uses: ./.github/actions/cleanup-java-env
        if: always()
  java_integration_tests_templates:
    name: Dataflow Templates Integration Tests
    needs: [java_integration_smoke_tests_templates]
    timeout-minutes: 240
    # Run on any runner that matches all the specified runs-on values.
    runs-on: [self-hosted, it]
    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Setup Environment
        id: setup-env
        uses: ./.github/actions/setup-env
      - name: Run Integration Tests
        run: |
          ./cicd/run-it-tests \
          --modules-to-build="DEFAULT" \
          --it-region="${{ env.IT_REGION }}" \
          --it-project="cloud-teleport-testing" \
          --it-integration-test-parallelism=6 \
          --it-thread-count=6 \
          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
          --it-private-connectivity="datastream-connect-2" \
          --it-base-container-image=":dev-image-latest" \
          --it-base-python-container-image=":dev-image-latest"
      - name: Upload Integration Tests Report
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always() # always run even if the previous step fails
        with:
          name: surefire-integration-test-results
          path: |
            **/surefire-reports/TEST-*.xml
            **/surefire-reports/*.html
            **/surefire-reports/html/**
          retention-days: 1
      - name: Integration Test report on GitHub
        uses: dorny/test-reporter@d1b5dff021aa968b288b02285a8bf47471c5928a # v3.2.0
        if: always()
        with:
          name: Integration Test report on GitHub
          path: '**/surefire-reports/TEST-*.xml'
          reporter: java-junit
          only-summary: 'true'
          token: ${{ secrets.GITHUB_TOKEN }}
          fail-on-error: 'false'
          list-suites: 'failed'
          list-tests: 'failed'
      - name: Cleanup Java Environment
        uses: ./.github/actions/cleanup-java-env
        if: always()
  java_load_tests_templates:
    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
    name: Dataflow Templates Load Tests
    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
    timeout-minutes: 600
    # Run on any runner that matches all the specified runs-on values.
    runs-on: [self-hosted, perf]
    steps:
    - name: Checkout Code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      with:
        persist-credentials: false
    - name: Setup Environment
      id: setup-env
      uses: ./.github/actions/setup-env
    - name: Run Load Tests
      run: |
        ./cicd/run-load-tests \
        --it-region="${{ env.IT_REGION }}" \
        --it-project="cloud-teleport-testing" \
        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
        --it-private-connectivity="datastream-connect-2"
    - name: Cleanup Java Environment
      uses: ./.github/actions/cleanup-java-env
      if: always()