Skip to content

[CI] Route BigQuery read tests by size #40858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bvolpato wants to merge 1 commit into
base: master
Choose a base branch
from
+40 −7
Draft

[CI] Route BigQuery read tests by size #40858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'schedule' && github.repository == 'apache/beam')
runs-on: [self-hosted, ubuntu-24.04, main]
timeout-minutes: 100
timeout-minutes: 480
name: ${{ matrix.job_name }}
strategy:
matrix:
Expand All @@ -75,12 +75,14 @@ jobs:
arguments: |
--testBigQueryTable=bqio_write_10GB_java_avro_$(date '+%m%d%H%M%S' --utc)
# The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
- name: run Java BigQueryIO Batch Avro Performance Test
- name: run Java BigQueryIO Batch Avro Performance Tests
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:io:bigquery-io-perf-tests:integrationTest
gradle-command: >-
:sdks:java:io:bigquery-io-perf-tests:integrationTest
--tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT
:runners:google-cloud-dataflow-java:bigQueryReadPerformanceTest
arguments: |
--tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \
--info \
-DintegrationTestRunner=dataflow \
-DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Batch_Java_Avro_test_arguments_1 }}]' \
Expand Down
27 changes: 26 additions & 1 deletion runners/google-cloud-dataflow-java/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,6 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test, dependsOn: copyG
])

include '**/*IT.class'
exclude '**/BigQueryIOReadIT.class'
exclude '**/BigQueryIOStorageReadTableRowIT.class'
exclude '**/BigQueryIODynamicQueryIT.class'
exclude '**/BigQueryIODynamicReadIT.class'
Expand All @@ -781,6 +780,10 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test, dependsOn: copyG
useJUnit {
excludeCategories "org.apache.beam.sdk.testing.UsesKms"
filter {
// Smaller reads run in PreCommit; the 1 TB read runs in the performance suite.
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryReadEmpty'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1M'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T'
// Error handling tests, only needs to run on direct runner
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableReadIT.testE2EBigtableReadWithSkippingLargeRows'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyMutationFailures'
Expand Down Expand Up @@ -846,6 +849,10 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) {
testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs)
useJUnit {
filter {
// Smaller reads run in PreCommit; the 1 TB read runs in the performance suite.
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryReadEmpty'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1M'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T'
// Error handling tests, only needs to run on direct runner
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyMutationFailures'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyRowFailures'
Expand All @@ -856,6 +863,24 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) {
}
}

task bigQueryReadPerformanceTest(type: Test) {
group = "Verification"
description = "Runs the 1 TB BigQuery read using the Dataflow Portable Runner."
dependsOn buildAndPushDockerJavaContainer

systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions)

include '**/BigQueryIOReadIT.class'
maxParallelForks 4
classpath = configurations.googleCloudPlatformIntegrationTest
testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs)
useJUnit {
filter {
includeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T'
}
}
}

tasks.register('examplesJavaRunnerV2IntegrationTest', Test.class) {
group = "Verification"
dependsOn buildAndPushDockerJavaContainer
Expand Down
10 changes: 8 additions & 2 deletions sdks/java/io/google-cloud-platform/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,6 @@ task integrationTest(type: Test, dependsOn: processTestResources) {

include '**/*IT.class'
exclude '**/BigQueryIOIcebergManagedTableIT.class'
exclude '**/BigQueryIOReadIT.class'
exclude '**/BigQueryIOStorageQueryIT.class'
exclude '**/BigQueryIOStorageReadIT.class'
exclude '**/BigQueryIOStorageWriteIT.class'
Expand All @@ -227,6 +226,9 @@ task integrationTest(type: Test, dependsOn: processTestResources) {
useJUnit {
excludeCategories "org.apache.beam.sdk.testing.UsesKms"
filter {
// Larger reads run in the Dataflow integration and performance suites.
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1G'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T'
// https://github.com/apache/beam/issues/32071
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableReadIT.testE2EBigtableSegmentRead'
}
Expand Down Expand Up @@ -342,7 +344,6 @@ task integrationTestBigQuery(type: Test, dependsOn: processTestResources) {
outputs.upToDateWhen { false }
include '**/bigquery/**/*IT.class'
exclude '**/BigQueryIOIcebergManagedTableIT.class'
exclude '**/BigQueryIOReadIT.class'
exclude '**/BigQueryIOStorageQueryIT.class'
exclude '**/BigQueryIOStorageReadIT.class'
exclude '**/BigQueryIOStorageWriteIT.class'
Expand All @@ -354,6 +355,11 @@ task integrationTestBigQuery(type: Test, dependsOn: processTestResources) {
testClassesDirs = sourceSets.test.output.classesDirs
useJUnit {
excludeCategories "org.apache.beam.sdk.testing.UsesKms"
filter {
// Larger reads run in the Dataflow integration and performance suites.
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1G'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOReadIT.testBigQueryRead1T'
}
}
}

Expand Down
Loading