From d067e003473e9e8a048a12542874fd7ae8c7f879 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Wed, 28 Oct 2026 00:59:47 -0400 Subject: [PATCH] [CI] Route BigQuery read tests by size --- ...rmanceTests_BigQueryIO_Batch_Java_Avro.yml | 10 ++++--- .../google-cloud-dataflow-java/build.gradle | 27 ++++++++++++++++++- .../io/google-cloud-platform/build.gradle | 10 +++++-- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml index eb706ae8bf1c..30a5710037a6 100644 --- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml @@ -52,7 +52,7 @@ jobs: github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'apache/beam') runs-on: [self-hosted, ubuntu-24.04, main] - timeout-minutes: 100 + timeout-minutes: 480 name: ${{ matrix.job_name }} strategy: matrix: @@ -75,12 +75,14 @@ jobs: arguments: | --testBigQueryTable=bqio_write_10GB_java_avro_$(date '+%m%d%H%M%S' --utc) # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - - name: run Java BigQueryIO Batch Avro Performance Test + - name: run Java BigQueryIO Batch Avro Performance Tests uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:bigquery-io-perf-tests:integrationTest + gradle-command: >- + :sdks:java:io:bigquery-io-perf-tests:integrationTest + --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT + :runners:google-cloud-dataflow-java:bigQueryReadPerformanceTest arguments: | - --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ --info \ -DintegrationTestRunner=dataflow \ -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Batch_Java_Avro_test_arguments_1 }}]' \ diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index d0aa2dfc0b10..ee06ba399e76 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -764,7 +764,6 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test, dependsOn: copyG ]) include '**/*IT.class' - exclude '**/BigQueryIOReadIT.class' exclude '**/BigQueryIOStorageReadTableRowIT.class' exclude '**/BigQueryIODynamicQueryIT.class' exclude '**/BigQueryIODynamicReadIT.class' @@ -781,6 +780,10 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test, dependsOn: copyG useJUnit { excludeCategories "org.apache.beam.sdk.testing.UsesKms" filter { + // Smaller reads run in PreCommit; the 1 TB read runs in the performance suite. + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryReadEmpty' + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1M' + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T' // Error handling tests, only needs to run on direct runner excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableReadIT.testE2EBigtableReadWithSkippingLargeRows' excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyMutationFailures' @@ -846,6 +849,10 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) { testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs) useJUnit { filter { + // Smaller reads run in PreCommit; the 1 TB read runs in the performance suite. + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryReadEmpty' + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1M' + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T' // Error handling tests, only needs to run on direct runner excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyMutationFailures' excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyRowFailures' @@ -856,6 +863,24 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) { } } +task bigQueryReadPerformanceTest(type: Test) { + group = "Verification" + description = "Runs the 1 TB BigQuery read using the Dataflow Portable Runner." + dependsOn buildAndPushDockerJavaContainer + + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions) + + include '**/BigQueryIOReadIT.class' + maxParallelForks 4 + classpath = configurations.googleCloudPlatformIntegrationTest + testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs) + useJUnit { + filter { + includeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T' + } + } +} + tasks.register('examplesJavaRunnerV2IntegrationTest', Test.class) { group = "Verification" dependsOn buildAndPushDockerJavaContainer diff --git a/sdks/java/io/google-cloud-platform/build.gradle b/sdks/java/io/google-cloud-platform/build.gradle index b1ad17c006df..5d4c57b9fa2d 100644 --- a/sdks/java/io/google-cloud-platform/build.gradle +++ b/sdks/java/io/google-cloud-platform/build.gradle @@ -212,7 +212,6 @@ task integrationTest(type: Test, dependsOn: processTestResources) { include '**/*IT.class' exclude '**/BigQueryIOIcebergManagedTableIT.class' - exclude '**/BigQueryIOReadIT.class' exclude '**/BigQueryIOStorageQueryIT.class' exclude '**/BigQueryIOStorageReadIT.class' exclude '**/BigQueryIOStorageWriteIT.class' @@ -227,6 +226,9 @@ task integrationTest(type: Test, dependsOn: processTestResources) { useJUnit { excludeCategories "org.apache.beam.sdk.testing.UsesKms" filter { + // Larger reads run in the Dataflow integration and performance suites. + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1G' + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T' // https://github.com/apache/beam/issues/32071 excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableReadIT.testE2EBigtableSegmentRead' } @@ -342,7 +344,6 @@ task integrationTestBigQuery(type: Test, dependsOn: processTestResources) { outputs.upToDateWhen { false } include '**/bigquery/**/*IT.class' exclude '**/BigQueryIOIcebergManagedTableIT.class' - exclude '**/BigQueryIOReadIT.class' exclude '**/BigQueryIOStorageQueryIT.class' exclude '**/BigQueryIOStorageReadIT.class' exclude '**/BigQueryIOStorageWriteIT.class' @@ -354,6 +355,11 @@ task integrationTestBigQuery(type: Test, dependsOn: processTestResources) { testClassesDirs = sourceSets.test.output.classesDirs useJUnit { excludeCategories "org.apache.beam.sdk.testing.UsesKms" + filter { + // Larger reads run in the Dataflow integration and performance suites. + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1G' + excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T' + } } }