From 2e174bf8be4c4c66135543d45f235359ea297106 Mon Sep 17 00:00:00 2001 From: Parv Arhatia Date: Mon, 5 Oct 2026 11:57:46 +0530 Subject: [PATCH] [#40250] Remove support for Python 3.10 --- .../test-properties.json | 16 +- .../setup-environment-action/action.yml | 4 +- .github/workflows/README.md | 2 +- .../beam_CloudML_Benchmarks_Dataflow.yml | 3 +- ...m_Inference_Python_Benchmarks_Dataflow.yml | 58 +++--- ..._LoadTests_Python_CoGBK_Dataflow_Batch.yml | 8 +- ...dTests_Python_CoGBK_Dataflow_Streaming.yml | 8 +- ...eam_LoadTests_Python_CoGBK_Flink_Batch.yml | 6 +- ...oadTests_Python_Combine_Dataflow_Batch.yml | 6 +- ...ests_Python_Combine_Dataflow_Streaming.yml | 6 +- ...m_LoadTests_Python_Combine_Flink_Batch.yml | 6 +- ...adTests_Python_Combine_Flink_Streaming.yml | 4 +- ...ests_Python_FnApiRunner_Microbenchmark.yml | 2 +- ...am_LoadTests_Python_GBK_Dataflow_Batch.yml | 10 +- ...oadTests_Python_GBK_Dataflow_Streaming.yml | 2 +- ...ts_Python_GBK_reiterate_Dataflow_Batch.yml | 4 +- ...ython_GBK_reiterate_Dataflow_Streaming.yml | 4 +- ..._LoadTests_Python_ParDo_Dataflow_Batch.yml | 8 +- ...dTests_Python_ParDo_Dataflow_Streaming.yml | 8 +- ...eam_LoadTests_Python_ParDo_Flink_Batch.yml | 6 +- ...LoadTests_Python_ParDo_Flink_Streaming.yml | 10 +- ...dTests_Python_SideInput_Dataflow_Batch.yml | 20 +- .../workflows/beam_LoadTests_Python_Smoke.yml | 4 +- ...erformanceTests_BiqQueryIO_Read_Python.yml | 2 +- ...nceTests_BiqQueryIO_Write_Python_Batch.yml | 2 +- ...manceTests_PubsubIOIT_Python_Streaming.yml | 2 +- ...ormanceTests_SpannerIO_Read_2GB_Python.yml | 2 +- ...Tests_SpannerIO_Write_2GB_Python_Batch.yml | 2 +- .../beam_PerformanceTests_TextIOIT_Python.yml | 2 +- ...rmanceTests_WordCountIT_PythonVersions.yml | 2 +- ..._PerformanceTests_xlang_KafkaIO_Python.yml | 2 +- .../workflows/beam_Playground_CI_Nightly.yml | 2 +- .../workflows/beam_Playground_Precommit.yml | 2 +- .../beam_PostCommit_PortableJar_Flink.yml | 4 +- .../beam_PostCommit_PortableJar_Spark.yml | 4 +- .github/workflows/beam_PostCommit_Python.yml | 2 +- .../workflows/beam_PostCommit_Python_Arm.yml | 2 +- .../beam_PostCommit_Python_Dependency.yml | 2 +- ...beam_PostCommit_Python_Examples_Direct.yml | 2 +- .../beam_PostCommit_Python_Examples_Flink.yml | 2 +- .../beam_PostCommit_Python_Examples_Spark.yml | 2 +- .../beam_PostCommit_Python_Nexmark_Direct.yml | 2 +- .../beam_PostCommit_Python_Portable_Flink.yml | 6 +- ...mit_Python_ValidatesContainer_Dataflow.yml | 2 +- ...on_ValidatesContainer_Dataflow_With_RC.yml | 2 +- ...Commit_Python_ValidatesRunner_Dataflow.yml | 2 +- ...ostCommit_Python_ValidatesRunner_Flink.yml | 2 +- ...ostCommit_Python_ValidatesRunner_Spark.yml | 2 +- ...m_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 2 +- ...eam_PostCommit_Python_Xlang_Gcp_Direct.yml | 2 +- ...am_PostCommit_Python_Xlang_IO_Dataflow.yml | 2 +- ...beam_PostCommit_Python_Xlang_IO_Direct.yml | 2 +- ...stCommit_Python_Xlang_Messaging_Direct.yml | 2 +- ...eam_PostCommit_TransformService_Direct.yml | 4 +- .../workflows/beam_PostCommit_XVR_Direct.yml | 4 +- .../workflows/beam_PostCommit_XVR_Flink.yml | 4 +- ...ostCommit_XVR_JavaUsingPython_Dataflow.yml | 4 +- ...ostCommit_XVR_PythonUsingJava_Dataflow.yml | 4 +- .../workflows/beam_PostCommit_XVR_Spark3.yml | 4 +- .../beam_PreCommit_Flink_Container.yml | 4 +- .../beam_PreCommit_Portable_Python.yml | 4 +- .../workflows/beam_PreCommit_Prism_Python.yml | 4 +- .github/workflows/beam_PreCommit_Python.yml | 2 +- .../workflows/beam_PreCommit_PythonDocker.yml | 2 +- .../beam_PreCommit_Python_Coverage.yml | 4 +- .../beam_PreCommit_Python_Dataframes.yml | 2 +- .../beam_PreCommit_Python_Examples.yml | 2 +- .../beam_PreCommit_Python_Integration.yml | 2 +- .../workflows/beam_PreCommit_Python_ML.yml | 4 +- .../beam_PreCommit_Python_Runners.yml | 2 +- .../beam_PreCommit_Python_Transforms.yml | 2 +- ...m_PreCommit_Xlang_Generated_Transforms.yml | 2 +- .../beam_Publish_Beam_SDK_Snapshots.yml | 3 - .../beam_Python_CostBenchmarks_Dataflow.yml | 6 +- ...Python_ValidatesContainer_Dataflow_ARM.yml | 2 +- .github/workflows/build_release_candidate.yml | 12 +- .github/workflows/build_wheels.yml | 4 +- .github/workflows/dask_runner_tests.yml | 4 +- ...n_CoGBK_Flink_Batch_100b_Multiple_Keys.txt | 2 +- ...thon_CoGBK_Flink_Batch_100b_Single_Key.txt | 2 +- .../python_CoGBK_Flink_Batch_10kB.txt | 2 +- ...ombine_Flink_Batch_2GB_10_byte_records.txt | 2 +- ...ython_Combine_Flink_Batch_2GB_Fanout_4.txt | 2 +- ...ython_Combine_Flink_Batch_2GB_Fanout_8.txt | 2 +- ...ne_Flink_Streaming_2GB_10_byte_records.txt | 2 +- ...n_Combine_Flink_Streaming_2GB_Fanout_4.txt | 2 +- ...n_Combine_Flink_Streaming_2GB_Fanout_8.txt | 2 +- ...Combine_Flink_Streaming_small_Fanout_1.txt | 2 +- ...Combine_Flink_Streaming_small_Fanout_2.txt | 2 +- ...on_GBK_Flink_Batch_2GB_of_100B_records.txt | 2 +- ...hon_GBK_Flink_Batch_2GB_of_10B_records.txt | 2 +- ...4_times_with_2GB_10-byte_records_total.txt | 2 +- ...8_times_with_2GB_10-byte_records_total.txt | 2 +- ...nk_Batch_reiterate_4_times_10kB_values.txt | 2 +- .../python_ParDo_Flink_Batch_10_Counters.txt | 2 +- ...python_ParDo_Flink_Batch_10_Iterations.txt | 2 +- ...ython_ParDo_Flink_Batch_200_Iterations.txt | 2 +- ...hon_ParDo_Flink_Streaming_100_Counters.txt | 2 +- ...thon_ParDo_Flink_Streaming_10_Counters.txt | 2 +- ...on_ParDo_Flink_Streaming_10_Iterations.txt | 2 +- ...n_ParDo_Flink_Streaming_200_Iterations.txt | 2 +- ...hon_ParDo_Flink_Streaming_5_Iterations.txt | 2 +- .github/workflows/python_dependency_tests.yml | 1 - .github/workflows/python_tests.yml | 3 +- .../republish_released_docker_containers.yml | 6 +- .github/workflows/run_perf_alert_tool.yml | 2 +- ...run_rc_validation_python_mobile_gaming.yml | 2 +- .github/workflows/typescript_tests.yml | 4 +- .../workflows/update_python_dependencies.yml | 1 - .test-infra/mock-apis/pyproject.toml | 2 +- .test-infra/tools/python_installer.sh | 2 +- CHANGES.md | 2 + build.gradle.kts | 29 +-- .../beam/gradle/BeamModulePlugin.groovy | 9 +- contributor-docs/release-guide.md | 2 +- gradle.properties | 2 +- local-env-setup.sh | 4 +- .../backend/containers/python/Dockerfile | 2 +- .../cloudbuild/playground_cd_examples.sh | 8 +- .../cloudbuild/playground_ci_examples.sh | 16 +- release/src/main/Dockerfile | 5 +- .../python_release_automation.sh | 2 +- .../runners/dataflow/DataflowRunnerTest.java | 12 +- sdks/python/apache_beam/__init__.py | 2 +- .../vertex_ai_custom_prediction/Dockerfile | 2 +- .../runners/dataflow/internal/apiclient.py | 4 +- .../dataflow/internal/apiclient_test.py | 6 +- .../apache_beam/testing/benchmarks/README.md | 2 +- sdks/python/build.gradle | 2 +- sdks/python/container/Dockerfile | 2 +- .../container/run_generate_requirements.sh | 4 +- .../expansion-service-container/Dockerfile | 4 +- sdks/python/pyproject.toml | 4 +- sdks/python/setup.py | 6 +- .../python/test-suites/dataflow/common.gradle | 2 +- sdks/python/test-suites/gradle.properties | 24 +-- .../python/test-suites/tox/py311/build.gradle | 186 ++++++++++++++++++ sdks/python/tox.ini | 48 ++--- .../en/documentation/runtime/environments.md | 4 +- .../sdks/python-sdk-image-build.md | 6 +- 140 files changed, 500 insertions(+), 336 deletions(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 5a1057470b68..56b93da21db3 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -1,15 +1,15 @@ { "PythonTestProperties": { - "ALL_SUPPORTED_VERSIONS": ["3.10", "3.11", "3.12", "3.13", "3.14"], - "LOWEST_SUPPORTED": ["3.10"], + "ALL_SUPPORTED_VERSIONS": ["3.11", "3.12", "3.13", "3.14"], + "LOWEST_SUPPORTED": ["3.11"], "HIGHEST_SUPPORTED": ["3.14"], - "ESSENTIAL_VERSIONS": ["3.10", "3.14"], - "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.10", "3.12", "3.13", "3.14"], + "ESSENTIAL_VERSIONS": ["3.11", "3.14"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.11", "3.12", "3.13", "3.14"], "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"], - "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.10", "3.11", "3.12", "3.13", "3.14"], - "LOAD_TEST_PYTHON_VERSION": "3.10", - "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.10", - "DEFAULT_INTERPRETER": "python3.10", + "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.11", "3.12", "3.13", "3.14"], + "LOAD_TEST_PYTHON_VERSION": "3.11", + "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.11", + "DEFAULT_INTERPRETER": "python3.11", "TOX_ENV": ["Cloud", "Cython"] }, "JavaTestProperties": { diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index 5c2afb39437d..b26d47fcc124 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -50,7 +50,7 @@ runs: if: ${{ inputs.python-version != '' }} uses: actions/setup-python@v6 with: - python-version: ${{ inputs.python-version == 'default' && '3.10' || inputs.python-version }} + python-version: ${{ inputs.python-version == 'default' && '3.11' || inputs.python-version }} cache: ${{ inputs.python-cache && 'pip' || 'none' }} cache-dependency-path: | sdks/python/setup.py @@ -68,7 +68,7 @@ runs: PYTHON_VERSION_INPUT: ${{ inputs.python-version }} run: | if [[ "$PYTHON_VERSION_INPUT" == "default" ]]; then - sanitized="310" + sanitized="311" else sanitized=$(printf '%s' "$PYTHON_VERSION_INPUT" | tr -s '[:space:]' '-' | sed 's/^-//;s/-$//') fi diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a45f11c6596f..f331c7af6592 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -284,7 +284,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour | [ PreCommit Integration and Load Test Framework ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_ItFramework.yml) | N/A |`beam_PreCommit_ItFramework.json`| [![.github/workflows/beam_PreCommit_ItFramework.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_ItFramework.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_ItFramework.yml?query=event%3Aschedule) | | [ PreCommit Java PVR Prism Loopback ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml) | N/A |`beam_PreCommit_Java_PVR_Prism_Loopback.json`| [![.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml?query=event%3Aschedule) | | [ PreCommit Java Solace IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solace_IO_Direct.yml) | N/A |`beam_PreCommit_Java_Solace_IO_Direct.json`| [![.github/workflows/beam_PreCommit_Java_Solace_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solace_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solace_IO_Direct.yml?query=event%3Aschedule) | -| [ PreCommit Prism Python ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Prism_Python.yml) | ['3.10', '3.13'] |`beam_PreCommit_Prism_Python.json`| [![.github/workflows/beam_PreCommit_Prism_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Prism_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Prism_Python.yml?query=event%3Aschedule) | +| [ PreCommit Prism Python ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Prism_Python.yml) | ['3.11', '3.14'] |`beam_PreCommit_Prism_Python.json`| [![.github/workflows/beam_PreCommit_Prism_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Prism_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Prism_Python.yml?query=event%3Aschedule) | | [ PreCommit Python Dill tests with dill deps installed ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Dill.yml) | ['3.12'] |`beam_PreCommit_Python_Dill.json`| [![.github/workflows/beam_PreCommit_Python_Dill.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Dill.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Dill.yml?query=event%3Aschedule) | Additional PreCommit jobs running basic SDK unit test on a matrices of operating systems. diff --git a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml index b852e41d16bc..995d817bb882 100644 --- a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml @@ -72,7 +72,6 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.10 3.11 - name: Prepare test arguments uses: ./.github/actions/test-arguments-action @@ -87,6 +86,6 @@ jobs: with: gradle-command: :sdks:python:test-suites:dataflow:tftTests arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -Prunner=DataflowRunner \ '-Popts=${{ env.beam_CloudML_Benchmarks_Dataflow_test_arguments_1 }}' diff --git a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml index 563d37832175..d86169533d2b 100644 --- a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml +++ b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml @@ -69,9 +69,9 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: '3.10' + python-version: '3.11' - name: Package Python SDK using Gradle - run: ./gradlew :sdks:python:sdist -PpythonVersion=3.10 + run: ./gradlew :sdks:python:sdist -PpythonVersion=3.11 - name: Configure Docker for Artifact Registry run: gcloud auth configure-docker us-docker.pkg.dev - name: Prepare test arguments @@ -117,7 +117,7 @@ jobs: -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.vllm_gemma_benchmarks \ -Prunner=DataflowRunner \ -PsdkLocationOverride=false \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/vllm_tests_requirements.txt '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_8 }} --mode=batch --job_name=benchmark-tests-vllm-with-gemma-2b-it-batch-${{env.NOW_UTC}} --sdk_container_image=${{ steps.build_vllm_image.outputs.image_url }}' - name: run Pytorch Sentiment Streaming using Hugging Face distilbert-base-uncased model uses: ./.github/actions/gradle-command-self-hosted-action @@ -127,7 +127,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_6 }} --mode=streaming --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-streaming-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \ - name: run Pytorch Sentiment Batch using Hugging Face distilbert-base-uncased model @@ -138,7 +138,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_7 }} --mode=batch --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-batch-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \ - name: run Pytorch Vision Classification with Resnet 101 @@ -149,7 +149,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-pytorch-imagenet-python-101-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet101-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Imagenet Classification with Resnet 152 @@ -160,7 +160,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Language Modeling using Hugging Face bert-base-uncased model @@ -171,7 +171,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_3 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-base-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Langauge Modeling using Hugging Face bert-large-uncased model @@ -182,7 +182,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_4 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-large-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased-${{env.NOW_UTC}}.txt' \ - name: run Pytorch Imagenet Classification with Resnet 152 with Tesla T4 GPU @@ -193,7 +193,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_5 }} --job_name=benchmark-tests-pytorch-imagenet-python-gpu-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152_gpu-${{env.NOW_UTC}}.txt' \ - name: run Table Row Inference Sklearn Batch @@ -204,7 +204,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.table_row_inference_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/table_row_inference_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_9 }} --autoscaling_algorithm=NONE --metrics_table=result_table_row_inference_batch --influx_measurement=result_table_row_inference_batch --mode=batch --input_file=gs://apache-beam-ml/testing/inputs/table_rows_100k_benchmark.jsonl --input_expand_factor=100 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_batch_outputs --job_name=benchmark-tests-table-row-inference-batch-${{env.NOW_UTC}}' \ - name: run Table Row Inference Sklearn Stream @@ -215,7 +215,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.table_row_inference_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/table_row_inference_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_10 }} --autoscaling_algorithm=THROUGHPUT_BASED --max_num_workers=20 --metrics_table=result_table_row_inference_stream --influx_measurement=result_table_row_inference_stream --mode=streaming --input_subscription=projects/apache-beam-testing/subscriptions/table_row_inference_benchmark --window_size_sec=60 --trigger_interval_sec=30 --timeout_ms=900000 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_stream_outputs --job_name=benchmark-tests-table-row-inference-stream-${{env.NOW_UTC}}' \ - name: run PyTorch Image Object Detection Faster R-CNN ResNet-50 Batch CPU @@ -226,7 +226,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_object_detection_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_object_detection_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --device=CPU --mode=batch --job_name=benchmark-tests-pytorch-image-object-detection-batch-${{env.NOW_UTC}} --metrics_table=torch_inference_image_object_detection_batch_cpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_object_detection_batch_cpu' \ - name: run PyTorch Image Captioning BLIP + CLIP Batch CPU @@ -237,7 +237,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_captioning_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_captioning_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_13 }} --device=CPU --mode=batch --job_name=benchmark-tests-pytorch-image-captioning-batch-${{env.NOW_UTC}} --metrics_table=torch_inference_image_captioning_batch_cpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_captioning_batch_cpu' - name: run PyTorch Image Object Detection Faster R-CNN ResNet-50 Batch GPU @@ -248,7 +248,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_object_detection_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_object_detection_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --device=GPU --worker_accelerator=type=nvidia-tesla-t4,count=1,install-nvidia-driver=true --mode=batch --job_name=benchmark-tests-pytorch-image-object-detection-batch-${{env.NOW_UTC}} --metrics_table=torch_inference_image_object_detection_batch_gpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_object_detection_batch_gpu' \ - name: run PyTorch Image Captioning BLIP + CLIP Batch GPU @@ -259,7 +259,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_captioning_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_captioning_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_13 }} --device=GPU --worker_accelerator=type=nvidia-tesla-t4,count=1,install-nvidia-driver=true --mode=batch --job_name=benchmark-tests-pytorch-image-captioning-batch-${{env.NOW_UTC}} --metrics_table=torch_inference_image_captioning_batch_gpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_captioning_batch_gpu' \ - name: run PyTorch Image Object Detection Faster R-CNN ResNet-50 Streaming CPU @@ -270,7 +270,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_object_detection_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_object_detection_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --device=CPU --mode=streaming --job_name=benchmark-tests-pytorch-image-object-detection-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_image_object_detection_streaming_cpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_object_detection_streaming_cpu' \ - name: run PyTorch Image Captioning BLIP + CLIP Streaming CPU @@ -281,7 +281,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_captioning_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_captioning_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_13 }} --device=CPU --mode=streaming --job_name=benchmark-tests-pytorch-image-captioning-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_image_captioning_streaming_cpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_captioning_streaming_cpu' - name: run PyTorch Image Object Detection Faster R-CNN ResNet-50 Streaming GPU @@ -292,7 +292,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_object_detection_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_object_detection_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --device=GPU --worker_accelerator=type=nvidia-tesla-t4,count=1,install-nvidia-driver=true --mode=streaming --job_name=benchmark-tests-pytorch-image-object-detection-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_image_object_detection_streaming_gpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_object_detection_streaming_gpu' \ - name: run PyTorch Image Captioning BLIP + CLIP Streaming GPU @@ -303,7 +303,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_captioning_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_image_captioning_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_13 }} --device=GPU --worker_accelerator=type=nvidia-tesla-t4,count=1,install-nvidia-driver=true --mode=streaming --job_name=benchmark-tests-pytorch-image-captioning-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_image_captioning_streaming_gpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_image_captioning_streaming_gpu' \ - name: run PyTorch Image Classification EfficientNet-B0 Streaming (Right-fitting) CPU @@ -314,7 +314,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_imagenet_rightfit_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_rightfit_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --device=CPU --experiments=enable_streaming_rightfitting --mode=streaming --job_name=benchmark-tests-pytorch-imagenet-rightfit-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_imagenet_stream_rightfit_cpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_imagenet_stream_rightfit_cpu' \ - name: run PyTorch Image Classification EfficientNet-B0 Streaming (Right-fitting Exactly-once) CPU @@ -325,7 +325,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_imagenet_rightfit_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_rightfit_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --device=CPU --experiments=enable_streaming_rightfitting --mode=streaming --job_name=benchmark-tests-pytorch-imagenet-rightfit-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_imagenet_stream_rightfit_once_cpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_imagenet_stream_rightfit_once_cpu' \ - name: run PyTorch Image Classification EfficientNet-B0 Streaming (Right-fitting) GPU @@ -336,7 +336,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_imagenet_rightfit_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_rightfit_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --device=GPU --experiments=enable_streaming_rightfitting,worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver:5xx --mode=streaming --job_name=benchmark-tests-pytorch-imagenet-rightfit-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_imagenet_stream_rightfit_gpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_imagenet_stream_rightfit_gpu' \ - name: run PyTorch Image Classification EfficientNet-B0 Streaming (Right-fitting Exactly-once) GPU @@ -347,7 +347,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_imagenet_rightfit_benchmarks \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/pytorch_rightfit_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --device=GPU --experiments=enable_streaming_rightfitting,worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver:5xx --mode=streaming --job_name=benchmark-tests-pytorch-imagenet-rightfit-streaming-${{env.NOW_UTC}} --metrics_table=torch_inference_imagenet_stream_rightfit_once_gpu --output_table=apache-beam-testing.beam_run_inference.result_torch_inference_imagenet_stream_rightfit_once_gpu' \ - name: run MLTransform Generate Vocab Batch @@ -358,7 +358,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_generate_vocab_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_generate_vocab_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_14 }} --job_name=benchmark-tests-mltransform-generate-vocab-batch-${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-perf-tests/mltransform/vocab_artifacts_${{env.NOW_UTC}} --output_vocab=gs://temp-storage-for-perf-tests/mltransform/vocab_outputs/mltransform_generate_vocab_batch_${{env.NOW_UTC}}' - name: run MLTransform One-Hot Encoding Batch @@ -369,7 +369,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_one_hot_encoding_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_one_hot_encoding_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_15 }} --autoscaling_algorithm=NONE --metrics_table=mltransform_one_hot_encoding_batch --influx_measurement=mltransform_one_hot_encoding_batch --job_name=benchmark-tests-mltransform-one-hot-encoding-batch-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/mltransform/one_hot_output_${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-end-to-end-tests/mltransform/artifacts_${{env.NOW_UTC}}' - name: run MLTransform Image Embedding GPU Batch @@ -380,7 +380,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_image_embedding_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_16 }} --artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_image_embedding_gpu/artifacts-${{env.NOW_UTC}} --job_name=benchmark-tests-mltransform-image-embedding-gpu-batch-${{env.NOW_UTC}}' - name: run MLTransform Text Embedding Batch @@ -391,7 +391,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_text_embedding_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_17 }} --output=gs://temp-storage-for-perf-tests/loadtests/mltransform_text_embedding/results-${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_text_embedding/artifacts-${{env.NOW_UTC}} --job_name=benchmark-tests-mltransform-text-embedding-batch-${{env.NOW_UTC}}' - name: run MLTransform Image Embedding CPU Batch @@ -402,6 +402,6 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_image_embedding_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_18 }} --artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_image_embedding_cpu/artifacts-${{env.NOW_UTC}} --job_name=benchmark-tests-mltransform-image-embedding-cpu-batch-${{env.NOW_UTC}}' diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml index f8181912221d..6cf5ffd08917 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml @@ -92,7 +92,7 @@ jobs: --info \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK 2GB of 100B records with multiple keys uses: ./.github/actions/gradle-command-self-hosted-action @@ -101,7 +101,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -110,7 +110,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -119,5 +119,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-cogbk-4-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml index ce0f92a94362..370b02a99a91 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK 2GB of 100B records with multiple keys uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,7 +100,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -109,7 +109,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ - name: run CoGBK reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -118,5 +118,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-cogbk-4-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml index d0484a9560e5..4a87fc0d40d6 100644 --- a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml @@ -102,7 +102,7 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | --info \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ @@ -112,7 +112,7 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | --info \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ @@ -122,7 +122,7 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | --info \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml index 8d61f85427a4..2b9b041a5b48 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml @@ -88,7 +88,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-combine-1-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 2 (fanout 4) uses: ./.github/actions/gradle-command-self-hosted-action @@ -97,7 +97,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-combine-2-${{env.NOW_UTC}}' \ - name: run Combine Dataflow Batch Python Load Test 3 (fanout 8) uses: ./.github/actions/gradle-command-self-hosted-action @@ -106,5 +106,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-combine-3-${{env.NOW_UTC}}' diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml index b2e5cf472186..29c1a7465003 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml @@ -88,7 +88,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-combine-1-${{env.NOW_UTC}}' \ - name: run 2GB Fanout 4 test uses: ./.github/actions/gradle-command-self-hosted-action @@ -97,7 +97,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-combine-4-${{env.NOW_UTC}}' \ - name: run 2GB Fanout 8 test uses: ./.github/actions/gradle-command-self-hosted-action @@ -106,5 +106,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-combine-5-${{env.NOW_UTC}}' diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml index bbb24314f7dc..87a7f7e26029 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml @@ -101,7 +101,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-combine-1-${{env.NOW_UTC}}' \ @@ -115,7 +115,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-combine-4-${{env.NOW_UTC}}' \ @@ -124,7 +124,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-combine-5-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml index 22ec2b33315f..205abf4418ae 100644 --- a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml @@ -103,7 +103,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \ @@ -112,7 +112,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ -Prunner=PortableRunner \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml index 30306cf957dd..d8ee66265c3a 100644 --- a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml +++ b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml @@ -83,5 +83,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.microbenchmarks_test \ -Prunner=DirectRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_FnApiRunner_Microbenchmark_test_arguments_1 }}' diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml index 56e13e623140..4b2688b49572 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml @@ -90,7 +90,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \ - name: run 2GB of 100B records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -99,7 +99,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \ - name: run 2GB of 100kB records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -108,7 +108,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \ - name: run fanout 4 times with 2GB 10-byte records test uses: ./.github/actions/gradle-command-self-hosted-action @@ -117,7 +117,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \ - name: run fanout 8 times with 2GB 10-byte records total test uses: ./.github/actions/gradle-command-self-hosted-action @@ -126,5 +126,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-gbk-5-${{env.NOW_UTC}}' diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml index be593ae7bd3e..09e0028edad2 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml @@ -86,7 +86,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-3-${{env.NOW_UTC}}' \ # // TODO(https://github.com/apache/beam/issues/20403). Skipping some cases because they are too slow: diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml index 994d2594d13a..b93dd193959d 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml @@ -87,7 +87,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \ - name: run reiterate 4 times 2MB values test uses: ./.github/actions/gradle-command-self-hosted-action @@ -96,5 +96,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-7-${{env.NOW_UTC}}' diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml index ab386a106ddd..e2f8daafd8f5 100644 --- a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml @@ -87,7 +87,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-6-${{env.NOW_UTC}}' \ - name: run reiterate 4 times 2MB values test uses: ./.github/actions/gradle-command-self-hosted-action @@ -96,5 +96,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-gbk-7-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml index f953f0f86197..eea28c2c2261 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,7 +100,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -109,7 +109,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Batch Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -118,5 +118,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml index 8f48cf022191..dad75aeaff19 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml @@ -91,7 +91,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -100,7 +100,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -109,7 +109,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Dataflow Streaming Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -118,5 +118,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml index ac1d3f0e5d3c..f853266ac08b 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml @@ -103,7 +103,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Batch Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -112,7 +112,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Batch Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -121,5 +121,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml index 20449dcb4060..ba836a577031 100644 --- a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml @@ -105,7 +105,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 2 (200 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -114,7 +114,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -123,7 +123,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_3 }} --job_name=load-tests-python-flink-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -132,7 +132,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_4 }} --job_name=load-tests-python-flink-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ - name: run ParDo Flink Streaming Python Load Test 5 (5 iterations) uses: ./.github/actions/gradle-command-self-hosted-action @@ -141,7 +141,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ -Prunner=PortableRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_5 }} --job_name=load-tests-python-flink-streaming-pardo-6-${{ steps.datetime.outputs.datetime }}' \ - name: Teardown Flink if: always() diff --git a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml index 9cbf795391ab..9812e86c710f 100644 --- a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml +++ b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml @@ -97,7 +97,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-sideinput-1-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 2 (1gb-1kb-10workers-1window-99key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -106,7 +106,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-sideinput-2-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 3 (10gb-1kb-10workers-1window-first-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -115,7 +115,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-sideinput-3-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 4 (10gb-1kb-10workers-1window-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -124,7 +124,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-sideinput-4-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 5 (1gb-1kb-10workers-1window-first-list) uses: ./.github/actions/gradle-command-self-hosted-action @@ -133,7 +133,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-sideinput-5-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 6 (1gb-1kb-10workers-1window-list) uses: ./.github/actions/gradle-command-self-hosted-action @@ -142,7 +142,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_6 }} --job_name=load-tests-python-dataflow-batch-sideinput-6-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 7 (1gb-1kb-10workers-1000window-1key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -151,7 +151,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_7 }} --job_name=load-tests-python-dataflow-batch-sideinput-7-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 8 (1gb-1kb-10workers-1000window-99key-percent-dict) uses: ./.github/actions/gradle-command-self-hosted-action @@ -160,7 +160,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_8 }} --job_name=load-tests-python-dataflow-batch-sideinput-8-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 9 (10gb-1kb-10workers-1000window-first-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -169,7 +169,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_9 }} --job_name=load-tests-python-dataflow-batch-sideinput-9-${{ steps.datetime.outputs.datetime }}' \ - name: run SideInput Dataflow Batch Python Load Test 10 (10gb-1kb-10workers-1000window-iterable) uses: ./.github/actions/gradle-command-self-hosted-action @@ -178,5 +178,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_10 }} --job_name=load-tests-python-dataflow-batch-sideinput-10-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_LoadTests_Python_Smoke.yml b/.github/workflows/beam_LoadTests_Python_Smoke.yml index bc6f7582b783..a425fa872c0f 100644 --- a/.github/workflows/beam_LoadTests_Python_Smoke.yml +++ b/.github/workflows/beam_LoadTests_Python_Smoke.yml @@ -86,7 +86,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DirectRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_1 }} --job_name=load-tests-python-direct-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ - name: run GroupByKey Python load test Dataflow uses: ./.github/actions/gradle-command-self-hosted-action @@ -95,5 +95,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml index 679b455098db..993dd86879be 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml @@ -85,6 +85,6 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | -PloadTest.mainClass=apache_beam.io.gcp.bigquery_read_perf_test \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1}}' diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml index 4a04465a4bb3..aa11c763e1c6 100644 --- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml @@ -85,6 +85,6 @@ jobs: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | -PloadTest.mainClass=apache_beam.io.gcp.bigquery_write_perf_test \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1}}' diff --git a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml index 9efa985e1bf6..8b2b84e8864e 100644 --- a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml +++ b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml @@ -86,5 +86,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.pubsub_io_perf_test \ -Prunner=TestDataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_PerformanceTests_PubsubIOIT_Python_Streaming_test_arguments_1 }}' diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml index 9a6d8f4d283c..446c59ea45d3 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml @@ -86,5 +86,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_read_perf_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Read_2GB_Python_test_arguments_1}}' diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml index 411c66092e04..04849544a3ee 100644 --- a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml @@ -86,5 +86,5 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_write_perf_test \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_test_arguments_1}}' diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml index 746db2490991..03f8b8db63e4 100644 --- a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml @@ -84,7 +84,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:load_tests:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.mainClass=apache_beam.io.filebasedio_perf_test \ -Prunner=DataflowRunner \ '-PloadTest.args=${{env.beam_PerformanceTests_TextIOIT_Python_test_arguments_1}}' diff --git a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml index f3d1cee493b0..e7da689c3ea6 100644 --- a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml +++ b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml @@ -60,7 +60,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PerformanceTests_WordCountIT_PythonVersions"] - python_version: ['3.10'] + python_version: ['3.11'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml index d872c1c6ebc5..f061132de101 100644 --- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -184,5 +184,5 @@ jobs: arguments: | -Prunner=DataflowRunner \ -PloadTest.mainClass=apache_beam.io.external.xlang_kafkaio_perf_test \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_PerfTests_xlang_KafkaIO_Python_test_arguments_1 }}' diff --git a/.github/workflows/beam_Playground_CI_Nightly.yml b/.github/workflows/beam_Playground_CI_Nightly.yml index 22006e03eed8..299e9e6a6d57 100644 --- a/.github/workflows/beam_Playground_CI_Nightly.yml +++ b/.github/workflows/beam_Playground_CI_Nightly.yml @@ -81,7 +81,7 @@ jobs: if: ${{ matrix.sdk == 'python' }} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:container:py310:docker -Pdocker-tag=nightly -Pdocker-pull-licences=true + gradle-command: :sdks:python:container:py311:docker -Pdocker-tag=nightly -Pdocker-pull-licences=true - name: Build SDK container uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_Playground_Precommit.yml b/.github/workflows/beam_Playground_Precommit.yml index f4dd7956a9df..c82251bd9ff3 100644 --- a/.github/workflows/beam_Playground_Precommit.yml +++ b/.github/workflows/beam_Playground_Precommit.yml @@ -43,7 +43,7 @@ jobs: job_name: [beam_Playground_PreCommit] env: DATASTORE_EMULATOR_VERSION: '423.0.0' - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' JAVA_VERSION: '17' steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml index bcc42689a7c9..311865192c0a 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml @@ -75,9 +75,9 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py310:testPipelineJarFlinkRunner + gradle-command: :sdks:python:test-suites:portable:py311:testPipelineJarFlinkRunner arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml index a6483c71b2d3..5eb1c8f790c6 100644 --- a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml +++ b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml @@ -75,9 +75,9 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py310:testPipelineJarSparkRunner + gradle-command: :sdks:python:test-suites:portable:py311:testPipelineJarSparkRunner arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml index fa23ecf44b50..63b0989fd38d 100644 --- a/.github/workflows/beam_PostCommit_Python.yml +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PostCommit_Python'] - python_version: ['3.10', '3.11', '3.12', '3.13','3.14'] + python_version: ['3.11', '3.12', '3.13','3.14'] os: [[self-hosted, ubuntu-24.04, highmem22]] if: | github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml index 3019b40d4ebd..eb308c21f7a1 100644 --- a/.github/workflows/beam_PostCommit_Python_Arm.yml +++ b/.github/workflows/beam_PostCommit_Python_Arm.yml @@ -63,7 +63,7 @@ jobs: fail-fast: false matrix: job_name: [beam_PostCommit_Python_Arm] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PostCommit_Python_Dependency.yml b/.github/workflows/beam_PostCommit_Python_Dependency.yml index 6300a04b2c0b..f11a111bbc30 100644 --- a/.github/workflows/beam_PostCommit_Python_Dependency.yml +++ b/.github/workflows/beam_PostCommit_Python_Dependency.yml @@ -58,7 +58,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PostCommit_Python_Dependency'] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] timeout-minutes: 360 if: | github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml index 89e9e78a8375..e1df08cb3705 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_Examples_Direct"] - python_version: ['3.10','3.11','3.12', '3.13','3.14'] + python_version: ['3.11','3.12', '3.13','3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml index 82483b1c2a74..51a690b48cfe 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_Examples_Flink"] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml index c8a44ac92623..99dcbd043057 100644 --- a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_Examples_Spark"] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml index d1fa140ac13d..f553073b78dc 100644 --- a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml @@ -129,7 +129,7 @@ jobs: with: gradle-command: :sdks:python:apache_beam:testing:benchmarks:nexmark:run arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ "-Pnexmark.args=${{ env.GRADLE_PYTHON_COMMAND_ARGUMENTS }} \ --query=${{ matrix.query }} \ --input=gs://temp-storage-for-perf-tests/nexmark/eventFiles/beam_PostCommit_Python_Nexmark_Direct/query${{ matrix.query }}-\*" diff --git a/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml b/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml index 11b90e659a13..95b21c27e785 100644 --- a/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_Portable_Flink.yml @@ -75,15 +75,15 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: '3.10' + python-version: '3.11' - name: Run flinkCompatibilityMatrix${{ matrix.environment_type }} script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py310:flinkCompatibilityMatrix${{ matrix.environment_type }} + gradle-command: :sdks:python:test-suites:portable:py311:flinkCompatibilityMatrix${{ matrix.environment_type }} arguments: | - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml index aae11370532b..8f684153dbd3 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow"] - python_version: ['3.10','3.11','3.12','3.13','3.14'] + python_version: ['3.11','3.12','3.13','3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml index 1a7b09af1b5f..f6031c1fe437 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC"] - python_version: ['3.10','3.11','3.12','3.13','3.14'] + python_version: ['3.11','3.12','3.13','3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml index e07dbd6ee1e6..2a156c0226d5 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Dataflow"] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml index f1eca9b26d57..57c82a366223 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Flink"] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml index 371a47e2146b..66ed0cff1d30 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PostCommit_Python_ValidatesRunner_Spark"] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] steps: - uses: actions/checkout@v7 with: diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index 51bb9b82f93a..47582312c4c5 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -73,7 +73,7 @@ jobs: with: java-version: default python-version: | - 3.10 + 3.11 3.14 - name: run PostCommit Python Xlang Gcp Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index da18136fb70a..c581a0983212 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -73,7 +73,7 @@ jobs: with: java-version: default python-version: | - 3.10 + 3.11 3.14 - name: Install docker compose run: | diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index eb19856a1e53..f54d10e47876 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -71,7 +71,7 @@ jobs: with: java-version: '17' python-version: | - 3.10 + 3.11 3.14 - name: run PostCommit Python Xlang IO Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml index 168d8cb9d7f9..7a8737952ec7 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml @@ -71,7 +71,7 @@ jobs: with: java-version: '17' python-version: | - 3.10 + 3.11 3.14 - name: run PostCommit Python Xlang IO Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Messaging_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Messaging_Direct.yml index 9641f19dd80f..427ca193d9bc 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Messaging_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Messaging_Direct.yml @@ -70,7 +70,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.10 + 3.11 3.14 - name: run PostCommit Python Xlang Messaging Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index 7e7c6e6c03ee..46d1fbea9423 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -60,7 +60,7 @@ jobs: strategy: matrix: job_name: ["beam_PostCommit_TransformService_Direct"] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] steps: - uses: actions/checkout@v7 with: @@ -74,7 +74,7 @@ jobs: # (BeamModulePlugin transformServiceJavaContainerSuffix). java-version: '17' python-version: | - 3.10 + 3.11 ${{ matrix.python_version }} - name: run TransformService Direct script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index b4e5ac2d1089..70dbba149367 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -60,7 +60,7 @@ jobs: strategy: matrix: job_name: ["beam_PostCommit_XVR_Direct"] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] steps: - uses: actions/checkout@v7 with: @@ -80,7 +80,7 @@ jobs: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.11' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml index c532be5ddd4b..7f55cb0e40cb 100644 --- a/.github/workflows/beam_PostCommit_XVR_Flink.yml +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -61,7 +61,7 @@ jobs: strategy: matrix: job_name: ["beam_PostCommit_XVR_Flink"] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] steps: - uses: actions/checkout@v7 with: @@ -81,7 +81,7 @@ jobs: gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.11' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml index 8f1b40f032e8..b1e001d2ad5c 100644 --- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -60,7 +60,7 @@ jobs: strategy: matrix: job_name: ["beam_PostCommit_XVR_JavaUsingPython_Dataflow"] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] steps: - uses: actions/checkout@v7 with: @@ -72,7 +72,7 @@ jobs: with: java-version: default python-version: | - 3.10 + 3.11 ${{ matrix.python_version }} - name: run PostCommit XVR JavaUsingPython Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml index 9fb50a21dc6e..26aaf22baf4c 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -60,7 +60,7 @@ jobs: strategy: matrix: job_name: ["beam_PostCommit_XVR_PythonUsingJava_Dataflow"] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] steps: - uses: actions/checkout@v7 with: @@ -72,7 +72,7 @@ jobs: with: java-version: default python-version: | - 3.10 + 3.11 ${{ matrix.python_version }} - name: run PostCommit XVR PythonUsingJava Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml index ea87a4aad4d6..82bffe04ff07 100644 --- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -60,7 +60,7 @@ jobs: strategy: matrix: job_name: ["beam_PostCommit_XVR_Spark3"] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] steps: - uses: actions/checkout@v7 with: @@ -80,7 +80,7 @@ jobs: gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ + -PskipNonPythonTask=${{ (matrix.python_version == '3.11' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Flink_Container.yml b/.github/workflows/beam_PreCommit_Flink_Container.yml index 8ca1ca6f7692..f6979cdf8829 100644 --- a/.github/workflows/beam_PreCommit_Flink_Container.yml +++ b/.github/workflows/beam_PreCommit_Flink_Container.yml @@ -77,7 +77,7 @@ env: ARTIFACTS_DIR: gs://beam-flink-cluster/beam-precommit-flink-container-${{ github.run_id }} DOCKER_REGISTRY: gcr.io DOCKER_REPOSITORY_ROOT: ${{ github.event_name == 'pull_request' && 'gcr.io/apache-beam-testing/beam-sdk-pr' || 'gcr.io/apache-beam-testing/beam-sdk' }} - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' PYTHON_SDK_IMAGE_TAG: latest jobs: @@ -114,7 +114,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:container:py310:docker + gradle-command: :sdks:python:container:py311:docker arguments: | -PpythonVersion=${{ env.PYTHON_VERSION }} \ -Pdocker-repository-root=${{ env.DOCKER_REPOSITORY_ROOT }} \ diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml b/.github/workflows/beam_PreCommit_Portable_Python.yml index b33371a6ac49..86d311ae0d95 100644 --- a/.github/workflows/beam_PreCommit_Portable_Python.yml +++ b/.github/workflows/beam_PreCommit_Portable_Python.yml @@ -79,7 +79,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Portable_Python'] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || @@ -97,7 +97,7 @@ jobs: java-version: default python-version: | ${{ matrix.python_version }} - 3.10 + 3.11 - name: Set PY_VER_CLEAN id: set_py_ver_clean run: | diff --git a/.github/workflows/beam_PreCommit_Prism_Python.yml b/.github/workflows/beam_PreCommit_Prism_Python.yml index 812a4af003b6..ed55aa1e88e7 100644 --- a/.github/workflows/beam_PreCommit_Prism_Python.yml +++ b/.github/workflows/beam_PreCommit_Prism_Python.yml @@ -73,7 +73,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Prism_Python'] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || @@ -91,7 +91,7 @@ jobs: java-version: default python-version: | ${{ matrix.python_version }} - 3.10 + 3.11 - name: Set PY_VER_CLEAN id: set_py_ver_clean run: | diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml index b89b8c558804..a58f70ef7c1a 100644 --- a/.github/workflows/beam_PreCommit_Python.yml +++ b/.github/workflows/beam_PreCommit_Python.yml @@ -80,7 +80,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python'] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml index 09954f067f9c..200ba655e36f 100644 --- a/.github/workflows/beam_PreCommit_PythonDocker.yml +++ b/.github/workflows/beam_PreCommit_PythonDocker.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ["beam_PreCommit_PythonDocker"] - python_version: ['3.10','3.11','3.12','3.13','3.14'] + python_version: ['3.11','3.12','3.13','3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index 4c371f5e70e8..fb53183780a9 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -62,7 +62,7 @@ jobs: fail-fast: false matrix: job_name: [beam_PreCommit_Python_Coverage] - python_version: ['3.10'] + python_version: ['3.11'] timeout-minutes: 180 if: | github.event_name == 'push' || @@ -89,7 +89,7 @@ jobs: TC_SLEEP_TIME: "1" uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py310:preCommitPyCoverage + gradle-command: :sdks:python:test-suites:tox:py311:preCommitPyCoverage - uses: codecov/codecov-action@v7 with: flags: python diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml index 434fe5c05187..2155aef5682d 100644 --- a/.github/workflows/beam_PreCommit_Python_Dataframes.yml +++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python_Dataframes'] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml index abf045bb03c3..00f80623300d 100644 --- a/.github/workflows/beam_PreCommit_Python_Examples.yml +++ b/.github/workflows/beam_PreCommit_Python_Examples.yml @@ -63,7 +63,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python_Examples'] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml index d83584aad160..3b2d24e26d0b 100644 --- a/.github/workflows/beam_PreCommit_Python_Integration.yml +++ b/.github/workflows/beam_PreCommit_Python_Integration.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python_Integration'] - python_version: ['3.10', '3.14'] + python_version: ['3.11', '3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml index cf8605627c2c..2f04755d7bca 100644 --- a/.github/workflows/beam_PreCommit_Python_ML.yml +++ b/.github/workflows/beam_PreCommit_Python_ML.yml @@ -63,7 +63,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python_ML'] - python_version: ['3.10','3.11','3.12','3.13'] + python_version: ['3.11','3.12','3.13'] # Run on both self-hosted and GitHub-hosted runners. # Some tests (marked require_docker_in_docker) can't run on Beam's # self-hosted runners due to Docker-in-Docker environment constraint. @@ -72,8 +72,6 @@ jobs: os: [[self-hosted, ubuntu-24.04, main], [ubuntu-latest]] exclude: # exercise DinD tests (needed for Milvus) for latest Python version - - python_version: '3.10' - os: [ubuntu-latest] - python_version: '3.11' os: [ubuntu-latest] - python_version: '3.12' diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml index c7b26dc4f732..c049ad26bc53 100644 --- a/.github/workflows/beam_PreCommit_Python_Runners.yml +++ b/.github/workflows/beam_PreCommit_Python_Runners.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python_Runners'] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml index 6f7d21d0bf2d..cd0b46d532f0 100644 --- a/.github/workflows/beam_PreCommit_Python_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml @@ -63,7 +63,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Python_Transforms'] - python_version: ['3.10','3.14'] + python_version: ['3.11','3.14'] if: | github.event_name == 'push' || github.event_name == 'pull_request' || diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml index c3ad7311ae91..b3d71a3fbde4 100644 --- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml @@ -81,7 +81,7 @@ jobs: fail-fast: false matrix: job_name: ['beam_PreCommit_Xlang_Generated_Transforms'] - python_version: ['3.10'] + python_version: ['3.11'] if: | github.event_name == 'push' || github.event_name == 'workflow_dispatch' || diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 7e5c5f287bf8..a2d2fe1d45ee 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -57,17 +57,14 @@ jobs: container_task: - "go:container:docker" - "java:container:pushAll" - - "python:container:py310:docker" - "python:container:py311:docker" - "python:container:py312:docker" - "python:container:py313:docker" - "python:container:py314:docker" - - "python:container:distroless:py310:docker" - "python:container:distroless:py311:docker" - "python:container:distroless:py312:docker" - "python:container:distroless:py313:docker" - "python:container:distroless:py314:docker" - - "python:container:ml:py310:docker" - "python:container:ml:py311:docker" - "python:container:ml:py312:docker" - "python:container:ml:py313:docker" diff --git a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml index b5b4aea84bfc..aa7be30a62b3 100644 --- a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml +++ b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml @@ -68,7 +68,7 @@ jobs: - name: Setup Python environment uses: ./.github/actions/setup-environment-action with: - python-version: '3.10' + python-version: '3.11' - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: @@ -88,7 +88,7 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.wordcount.wordcount \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/testing/benchmarks/wordcount/requirements.txt \ '-PloadTest.args=${{ env.beam_Python_Cost_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-wordcount-python-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_wordcount-${{env.NOW_UTC}}.txt' \ - name: Run Tensorflow MNIST Image Classification on Dataflow @@ -99,6 +99,6 @@ jobs: arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.tensorflow_mnist_classification_cost_benchmark \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -PloadTest.requirementsTxtFile=apache_beam/ml/inference/tensorflow_tests_requirements.txt \ '-PloadTest.args=${{ env.beam_Python_Cost_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-tf-mnist-classification-python-${{env.NOW_UTC}} --input_file=gs://apache-beam-ml/testing/inputs/it_mnist_data.csv --output_file=gs://temp-storage-for-end-to-end-tests/inference/result_tf_mnist-${{env.NOW_UTC}}.txt --model=gs://apache-beam-ml/models/tensorflow/mnist/' \ diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml index c6a8f4be93e2..1e1ea21727bb 100644 --- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml +++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml @@ -53,7 +53,7 @@ jobs: fail-fast: false matrix: job_name: [beam_Python_ValidatesContainer_Dataflow_ARM] - python_version: ['3.10','3.11','3.12','3.13','3.14'] + python_version: ['3.11','3.12','3.13','3.14'] if: | github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 9efdaea4834e..d437b7a984a3 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -304,10 +304,10 @@ jobs: java-version: | 21 11 - - name: Install Python 3.10 + - name: Install Python 3.11 uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 - name: Remove default github maven configuration @@ -355,10 +355,10 @@ jobs: token: ${{ github.event.inputs.REPO_TOKEN }} ref: release-docs persist-credentials: true - - name: Install Python 3.10 + - name: Install Python 3.11 uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Install node uses: actions/setup-node@v7 with: @@ -602,10 +602,10 @@ jobs: path: beam token: ${{ github.event.inputs.REPO_TOKEN }} persist-credentials: true - - name: Install Python 3.10 + - name: Install Python 3.11 uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Install Java 17 uses: actions/setup-java@v6 with: diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index aa2c90103f02..c832dd582c64 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -50,7 +50,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} # Keep in sync with py_version matrix value below - if changed, change that as well. - PY_VERSIONS_FULL: "cp310-* cp311-* cp312-* cp313-* cp314-*" + PY_VERSIONS_FULL: "cp311-* cp312-* cp313-* cp314-*" outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }} @@ -238,7 +238,7 @@ jobs: {"os": "ubuntu-24.04", "runner": "ubuntu-22.04", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}", arch: "aarch64" } ] # Keep in sync (remove asterisks) with PY_VERSIONS_FULL env var above - if changed, change that as well. - py_version: ["cp310-", "cp311-", "cp312-", "cp313-", "cp314-"] + py_version: ["cp311-", "cp312-", "cp313-", "cp314-"] steps: - name: Download python source distribution from artifacts uses: actions/download-artifact@v8 diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml index 1b17bfb88e3d..5125c00d9588 100644 --- a/.github/workflows/dask_runner_tests.yml +++ b/.github/workflows/dask_runner_tests.yml @@ -46,7 +46,7 @@ jobs: - name: Install python uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Build source working-directory: ./sdks/python run: pip install -U build && python -m build --sdist @@ -67,7 +67,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] params: [ - {"py_ver": "3.10", "tox_env": "py310" }, + {"py_ver": "3.11", "tox_env": "py311" }, ] steps: - name: Checkout code diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt index dc851c279215..2b611fc8bf72 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt @@ -25,4 +25,4 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt index b462794a444e..4d66b54e1fc2 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt @@ -25,4 +25,4 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt index d8154c115405..fd490d8aec1f 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt @@ -25,4 +25,4 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt index dba7cf9c95d6..a66514b46cd1 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt @@ -22,6 +22,6 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt index c79db43476ad..70c91c576136 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=4 --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt index fb3e08de8916..266890b7e328 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=8 --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt index 925cfc75d760..02dab88ec24b 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt @@ -22,7 +22,7 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --top_count=20 --streaming --use_stateful_load_generator diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt index a89a46a3747c..8861a7512663 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=4 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt index 9edc487cdf16..d713b4b8a2fb 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=8 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt index 91420cb34be0..9b89c8a97322 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=1 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt index e8054ceec18f..b12f966b7323 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt @@ -22,7 +22,7 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=2 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt index ddf7a314c0d8..11e14ffb5763 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt index 29c5a085f5f6..6fb19aa6e6d6 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt index 34695a08f96e..bf0a17978a76 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,5 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt index 7a7db563a769..f76d6176b894 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,5 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt index 40db954bce78..1154da900faf 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt @@ -24,5 +24,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt index 2ab93cc13f5c..8ae1401bc2c7 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt index e0b9e1093716..d822b109ca96 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt index 2ab93cc13f5c..8ae1401bc2c7 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt @@ -25,5 +25,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt index 1265387706b3..d1386a9ce3e9 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt index d725a7d1b601..c54710fc6eb2 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt index 7f7f428c4081..de0a77fb62af 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt @@ -27,6 +27,6 @@ --stateful --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt index 83411793c2b2..6071a2b7404a 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt @@ -26,6 +26,6 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt index 14f94d641847..d306b8573de5 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt @@ -30,6 +30,6 @@ --shutdown_sources_after_idle_ms=300000 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml index 5a49d6bfd7f8..daa299804c5e 100644 --- a/.github/workflows/python_dependency_tests.yml +++ b/.github/workflows/python_dependency_tests.yml @@ -27,7 +27,6 @@ jobs: matrix: os: [ubuntu-latest] params: [ - {"py_ver": "3.10", "py_env": "py310" }, { "py_ver": "3.11", "py_env": "py311" }, { "py_ver": "3.12", "py_env": "py312" }, { "py_ver": "3.13", "py_env": "py313" }, diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 1f676921ece4..bd48ce8cca1e 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -97,7 +97,6 @@ jobs: matrix: os: [macos-latest, windows-latest] params: [ - { "py_ver": "3.10", "tox_env": "py310" }, { "py_ver": "3.11", "tox_env": "py311" }, { "py_ver": "3.12", "tox_env": "py312" }, { "py_ver": "3.13", "tox_env": "py313" }, @@ -140,7 +139,7 @@ jobs: fail-fast: false matrix: os: [[self-hosted, ubuntu-24.04, main], macos-latest, windows-latest] - python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python: ["3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout code uses: actions/checkout@v7 diff --git a/.github/workflows/republish_released_docker_containers.yml b/.github/workflows/republish_released_docker_containers.yml index 3fd368d74641..a54b8a6ab3d7 100644 --- a/.github/workflows/republish_released_docker_containers.yml +++ b/.github/workflows/republish_released_docker_containers.yml @@ -71,10 +71,10 @@ jobs: java-version: | 21 11 - - name: Install Python 3.10 + - name: Install Python 3.11 uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Authenticate on GCP uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 with: @@ -96,7 +96,7 @@ jobs: run: | ./gradlew ${{ matrix.images_to_publish.gradle_task }} \ -PisRelease \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ -Pdocker-pull-licenses \ -Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} \ -Pdocker-repository-root=gcr.io/apache-beam-testing/updated_released_container_images \ diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml index 9a2e3810391c..5b99eeaf7ae4 100644 --- a/.github/workflows/run_perf_alert_tool.yml +++ b/.github/workflows/run_perf_alert_tool.yml @@ -41,7 +41,7 @@ jobs: - name: Install python uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Install Apache Beam working-directory: ./sdks/python run: pip install -e .[gcp,test] diff --git a/.github/workflows/run_rc_validation_python_mobile_gaming.yml b/.github/workflows/run_rc_validation_python_mobile_gaming.yml index 61bd4b1fb68c..2b0ff8ac0e02 100644 --- a/.github/workflows/run_rc_validation_python_mobile_gaming.yml +++ b/.github/workflows/run_rc_validation_python_mobile_gaming.yml @@ -79,7 +79,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VER }} RC_NUM: ${{ github.event.inputs.RC_NUM }} RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}" - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' BEAM_PYTHON_SDK_TAR_GZ: apache_beam-${{ github.event.inputs.RELEASE_VER }}.tar.gz BEAM_SOURCE_ZIP: apache-beam-${{ github.event.inputs.RELEASE_VER }}-source-release.zip APACHE_DIST_URL_BASE: https://dist.apache.org/repos/dist/dev/beam/${{ github.event.inputs.RELEASE_VER }} diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml index 58cecd6164b2..e80fe8cea9c3 100644 --- a/.github/workflows/typescript_tests.yml +++ b/.github/workflows/typescript_tests.yml @@ -110,7 +110,7 @@ jobs: - name: Install Python uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Install Java uses: actions/setup-java@v6 with: @@ -180,7 +180,7 @@ jobs: - name: Install python uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: '3.11' - name: Install Java uses: actions/setup-java@v6 with: diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml index b934f402067c..8eef0a1a5a7f 100644 --- a/.github/workflows/update_python_dependencies.yml +++ b/.github/workflows/update_python_dependencies.yml @@ -61,7 +61,6 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: | - 3.10 3.11 3.12 3.13 diff --git a/.test-infra/mock-apis/pyproject.toml b/.test-infra/mock-apis/pyproject.toml index b04d106f8a45..d576d4b8abc3 100644 --- a/.test-infra/mock-apis/pyproject.toml +++ b/.test-infra/mock-apis/pyproject.toml @@ -27,7 +27,7 @@ packages = [ ] [tool.poetry.dependencies] -python = "^3.10" +python = "^3.11" google = "^3.0.0" grpcio = "^1.53.0" grpcio-tools = "^1.53.0" diff --git a/.test-infra/tools/python_installer.sh b/.test-infra/tools/python_installer.sh index a242e1335b01..18b2cda9a199 100644 --- a/.test-infra/tools/python_installer.sh +++ b/.test-infra/tools/python_installer.sh @@ -20,7 +20,7 @@ set -euo pipefail # Variable containing the python versions to install -python_versions_arr=("3.10.10" "3.11.4" "3.12.6" "3.13.9") +python_versions_arr=("3.11.4" "3.12.6" "3.13.9") # Install pyenv dependencies. pyenv_dep(){ diff --git a/CHANGES.md b/CHANGES.md index dbe5a0544556..5506c13d7843 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -78,6 +78,8 @@ ## Deprecations +* Support for Python 3.10 was removed following its end-of-life ([#40250](https://github.com/apache/beam/issues/40250)). + * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). ## Bugfixes diff --git a/build.gradle.kts b/build.gradle.kts index 5745a753b3ab..5a1de120a1d8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -498,7 +498,6 @@ tasks.register("playgroundPreCommit") { tasks.register("pythonPreCommit") { dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon") - dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310") dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311") dependsOn(":sdks:python:test-suites:tox:py312:preCommitPy312") dependsOn(":sdks:python:test-suites:tox:py313:preCommitPy313") @@ -515,7 +514,6 @@ tasks.register("pythonDocsPreCommit") { } tasks.register("pythonDockerBuildPreCommit") { - dependsOn(":sdks:python:container:py310:docker") dependsOn(":sdks:python:container:py311:docker") dependsOn(":sdks:python:container:py312:docker") dependsOn(":sdks:python:container:py313:docker") @@ -795,23 +793,12 @@ tasks.register("validateChanges") { } } -tasks.register("python310PostCommit") { - dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT") - dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT") - dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310") - dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest") - dependsOn(":sdks:python:test-suites:direct:py310:azureIntegrationTest") - // TODO: https://github.com/apache/beam/issues/22651 - // The default container uses Python 3.10. The goal here is to - // duild Docker images for TensorRT tests during run time for python versions - // other than 3.10 and add these tests in other python postcommit suites. - dependsOn(":sdks:python:test-suites:dataflow:py310:inferencePostCommitIT") -} tasks.register("python311PostCommit") { dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest") + dependsOn(":sdks:python:test-suites:direct:py311:azureIntegrationTest") dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311") } @@ -838,12 +825,12 @@ tasks.register("python314PostCommit") { } tasks.register("portablePythonPreCommit") { - dependsOn(":sdks:python:test-suites:portable:py310:preCommitPy310") + dependsOn(":sdks:python:test-suites:portable:py311:preCommitPy311") dependsOn(":sdks:python:test-suites:portable:py314:preCommitPy314") } tasks.register("pythonSparkPostCommit") { - dependsOn(":sdks:python:test-suites:portable:py310:sparkValidatesRunner") + dependsOn(":sdks:python:test-suites:portable:py311:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py314:sparkValidatesRunner") } @@ -867,15 +854,15 @@ tasks.register("javaExamplesDataflowPrecommit") { tasks.register("whitespacePreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py310:archiveFilesToLint") - dependsOn(":sdks:python:test-suites:tox:py310:unpackFilesToLint") - dependsOn(":sdks:python:test-suites:tox:py310:whitespacelint") + dependsOn(":sdks:python:test-suites:tox:py311:archiveFilesToLint") + dependsOn(":sdks:python:test-suites:tox:py311:unpackFilesToLint") + dependsOn(":sdks:python:test-suites:tox:py311:whitespacelint") } tasks.register("typescriptPreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py310:eslint") - dependsOn(":sdks:python:test-suites:tox:py310:jest") + dependsOn(":sdks:python:test-suites:tox:py311:eslint") + dependsOn(":sdks:python:test-suites:tox:py311:jest") } tasks.register("pushAllRunnersDockerImages") { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 023b6dbec477..1453b63b2f2f 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -3171,10 +3171,10 @@ class BeamModulePlugin implements Plugin { // set from commandline with -PpythonVersion, or in build script of certain project. // If none of them applied, version set here will be used as default value. project.ext.pythonVersion = project.hasProperty('pythonVersion') ? - project.pythonVersion : '3.10' + project.pythonVersion : '3.11' // Set min/max python versions used for containers and supported versions. - project.ext.minPythonVersion = 10 + project.ext.minPythonVersion = 11 project.ext.maxPythonVersion = 14 def setupVirtualenv = project.tasks.register('setupVirtualenv') { @@ -3249,7 +3249,7 @@ class BeamModulePlugin implements Plugin { def uvCacheDir = "${project.ext.envdir}/.uv_cache" project.exec { executable 'sh' - // Default uv cache is global; py310/py314 installGcpTest fight over the same lock. + // Default uv cache is global; py311/py314 installGcpTest fight over the same lock. args '-c', ". \"${project.ext.envdir}/bin/activate\" && pip install uv && uv pip install --cache-dir \"${uvCacheDir}\" --pre \"${installTargets}\"" } } @@ -3301,7 +3301,7 @@ class BeamModulePlugin implements Plugin { def actualToxEnv = tox_env def osName = System.getProperty("os.name").toLowerCase() if (osName.contains("mac")) { - // Only append -macos for standard python environments (py39, py310, etc.) + // Only append -macos for standard python environments (py311, py312, etc.) if (tox_env.matches("py\\d+")) { actualToxEnv = "${tox_env}-macos" } @@ -3391,7 +3391,6 @@ class BeamModulePlugin implements Plugin { mustRunAfter = [ ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar", ':runners:spark:3:job-server:shadowJar', - ':sdks:python:container:py310:docker', ':sdks:python:container:py311:docker', ':sdks:python:container:py312:docker', ':sdks:python:container:py313:docker', diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md index 270ede43ce89..25c783dfcf9f 100644 --- a/contributor-docs/release-guide.md +++ b/contributor-docs/release-guide.md @@ -582,7 +582,7 @@ with tags: `${RELEASE_VERSION}rc${RC_NUM}` Verify that third party licenses are included in Docker. You can do this with a simple script: RC_TAG=${RELEASE_VERSION}rc${RC_NUM} - for pyver in 3.10 3.11 3.12 3.13 3.14; do + for pyver in 3.11 3.12 3.13 3.14; do docker run --rm --entrypoint sh \ apache/beam_python${pyver}_sdk:${RC_TAG} \ -c 'ls -al /opt/apache/beam/third_party_licenses/ | wc -l' diff --git a/gradle.properties b/gradle.properties index dcf133bc81e5..34c5926be809 100644 --- a/gradle.properties +++ b/gradle.properties @@ -43,7 +43,7 @@ flink_versions=1.19,1.20,2.0,2.1,2.2 # supported spark versions spark_versions=3,4 # supported python versions -python_versions=3.10,3.11,3.12,3.13,3.14 +python_versions=3.11,3.12,3.13,3.14 # Maven Central fallback mirror URL mavenCentralMirrorUrl=https://maven-central.storage-download.googleapis.com/maven2/ diff --git a/local-env-setup.sh b/local-env-setup.sh index 0c28b4e65f82..d373cd71ea32 100755 --- a/local-env-setup.sh +++ b/local-env-setup.sh @@ -55,7 +55,7 @@ if [ "$kernelname" = "Linux" ]; then exit fi - for ver in 3.10 3.11 3.12 3.13 3.14 3; do + for ver in 3.11 3.12 3.13 3.14 3; do apt install --yes python$ver-venv done @@ -89,7 +89,7 @@ elif [ "$kernelname" = "Darwin" ]; then echo "Installing openjdk@8" brew install openjdk@8 fi - for ver in 3.10 3.11 3.12 3.13 3.14; do + for ver in 3.11 3.12 3.13 3.14; do if brew ls --versions python@$ver > /dev/null; then echo "python@$ver already installed. Skipping" brew info python@$ver diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile index 87d5bbf9f050..586e2ffd95c7 100644 --- a/playground/backend/containers/python/Dockerfile +++ b/playground/backend/containers/python/Dockerfile @@ -17,7 +17,7 @@ ############################################################################### ARG GO_BASE_IMAGE=golang:1-bookworm ARG SDK_TAG -ARG BASE_IMAGE=apache/beam_python3.10_sdk:$SDK_TAG +ARG BASE_IMAGE=apache/beam_python3.11_sdk:$SDK_TAG FROM $GO_BASE_IMAGE AS build ARG GIT_COMMIT="" ARG GIT_TIMESTAMP="0" diff --git a/playground/infrastructure/cloudbuild/playground_cd_examples.sh b/playground/infrastructure/cloudbuild/playground_cd_examples.sh index b32fc942ad27..05ff39dfbf9f 100644 --- a/playground/infrastructure/cloudbuild/playground_cd_examples.sh +++ b/playground/infrastructure/cloudbuild/playground_cd_examples.sh @@ -97,15 +97,15 @@ LogOutput "Installing python and dependencies." export DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https ca-certificates software-properties-common curl unzip apt-utils > /dev/null 2>&1 add-apt-repository -y ppa:deadsnakes/ppa > /dev/null 2>&1 && apt update > /dev/null 2>&1 -apt install -y python3.10 python3-distutils python3-pip > /dev/null 2>&1 +apt install -y python3.11 python3-distutils python3-pip > /dev/null 2>&1 apt install -y --reinstall python3-distutils > /dev/null 2>&1 apt install -y python3-virtualenv virtualenv play_venv source play_venv/bin/activate pip install --upgrade google-api-python-client > /dev/null 2>&1 -python3.10 -m pip install pip --upgrade > /dev/null 2>&1 -ln -s /usr/bin/python3.10 /usr/bin/python > /dev/null 2>&1 -apt install -y python3.10-venv > /dev/null 2>&1 +python3.11 -m pip install pip --upgrade > /dev/null 2>&1 +ln -s /usr/bin/python3.11 /usr/bin/python > /dev/null 2>&1 +apt install -y python3.11-venv > /dev/null 2>&1 LogOutput "Installing Python packages from beam/playground/infrastructure/requirements.txt" cd $BEAM_ROOT_DIR diff --git a/playground/infrastructure/cloudbuild/playground_ci_examples.sh b/playground/infrastructure/cloudbuild/playground_ci_examples.sh index d39e0a696c2a..bdddd0b75a82 100755 --- a/playground/infrastructure/cloudbuild/playground_ci_examples.sh +++ b/playground/infrastructure/cloudbuild/playground_ci_examples.sh @@ -94,12 +94,12 @@ export DEBIAN_FRONTEND=noninteractive LogOutput "Installing Python environment" apt-get install -y apt-transport-https ca-certificates software-properties-common curl unzip apt-utils > /dev/null add-apt-repository -y ppa:deadsnakes/ppa > /dev/null && apt update > /dev/null -apt install -y python3.10 python3-distutils python3-pip > /dev/null +apt install -y python3.11 python3-distutils python3-pip > /dev/null apt install --reinstall python3-distutils > /dev/null pip install --upgrade google-api-python-client > /dev/null -python3.10 -m pip install pip --upgrade > /dev/null -ln -s /usr/bin/python3.10 /usr/bin/python > /dev/null -apt install python3.10-venv > /dev/null +python3.11 -m pip install pip --upgrade > /dev/null +ln -s /usr/bin/python3.11 /usr/bin/python > /dev/null +apt install python3.11-venv > /dev/null LogOutput "Installing Python packages from beam/playground/infrastructure/requirements.txt" pip install -r $BEAM_ROOT_DIR/playground/infrastructure/requirements.txt @@ -191,13 +191,13 @@ do if [ "$sdk" == "python" ] then # Build fails without docker-pull-licenses=true in Cloud Build - LogOutput "Building Python base image container apache/beam_python3.10_sdk:$DOCKERTAG" - LogOutput "./gradlew -i :sdks:python:container:py310:docker -Pdocker-tag=$DOCKERTAG -Pdocker-pull-licenses=true" + LogOutput "Building Python base image container apache/beam_python3.11_sdk:$DOCKERTAG" + LogOutput "./gradlew -i :sdks:python:container:py311:docker -Pdocker-tag=$DOCKERTAG -Pdocker-pull-licenses=true" sdk_tag=$DOCKERTAG - ./gradlew -i :sdks:python:container:py310:docker -Pdocker-tag=$DOCKERTAG -Pdocker-pull-licenses=true + ./gradlew -i :sdks:python:container:py311:docker -Pdocker-tag=$DOCKERTAG -Pdocker-pull-licenses=true if [ $? -ne 0 ] then - LogOutput "Build failed for apache/beam_python3.10_sdk:$DOCKERTAG" + LogOutput "Build failed for apache/beam_python3.11_sdk:$DOCKERTAG" continue fi fi diff --git a/release/src/main/Dockerfile b/release/src/main/Dockerfile index 36171674d452..b01b48e66de7 100644 --- a/release/src/main/Dockerfile +++ b/release/src/main/Dockerfile @@ -43,11 +43,10 @@ RUN curl https://pyenv.run | bash && \ echo ''eval "$(pyenv init -)"'' >> /root/.bashrc && \ source /root/.bashrc && \ pyenv install 3.9.4 && \ - pyenv install 3.10.7 && \ pyenv install 3.11.3 && \ pyenv install 3.12.3 && \ pyenv install 3.13.9 && \ - pyenv global 3.10.7 3.11.3 3.12.3 3.13.9 + pyenv global 3.11.3 3.12.3 3.13.9 # Install a Go version >= 1.16 so we can bootstrap higher # Go versions @@ -90,4 +89,4 @@ RUN echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings RUN sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin # Copy Beam -RUN git clone https://github.com/apache/beam \ No newline at end of file +RUN git clone https://github.com/apache/beam diff --git a/release/src/main/python-release/python_release_automation.sh b/release/src/main/python-release/python_release_automation.sh index 8f4f509b6f1c..d7a9438d0383 100755 --- a/release/src/main/python-release/python_release_automation.sh +++ b/release/src/main/python-release/python_release_automation.sh @@ -19,7 +19,7 @@ source release/src/main/python-release/run_release_candidate_python_quickstart.sh source release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh -for version in 3.10 3.11 3.12 3.13 3.14 +for version in 3.11 3.12 3.13 3.14 do run_release_candidate_python_quickstart "tar" "python${version}" run_release_candidate_python_mobile_gaming "tar" "python${version}" diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java index 549e4f782a39..b196aa1714c6 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java @@ -1280,8 +1280,8 @@ private static RunnerApi.Pipeline containerUrlToPipeline(String url) { @Test public void testApplySdkEnvironmentOverrides() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); - String dockerHubPythonContainerUrl = "apache/beam_python3.10_sdk:latest"; - String gcrPythonContainerUrl = "gcr.io/apache-beam-testing/beam-sdk/beam_python3.10_sdk:latest"; + String dockerHubPythonContainerUrl = "apache/beam_python3.11_sdk:latest"; + String gcrPythonContainerUrl = "gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest"; options.setSdkHarnessContainerImageOverrides(".*python.*," + gcrPythonContainerUrl); DataflowRunner runner = DataflowRunner.fromOptions(options); RunnerApi.Pipeline pipeline = containerUrlToPipeline(dockerHubPythonContainerUrl); @@ -1292,8 +1292,8 @@ public void testApplySdkEnvironmentOverrides() throws IOException { @Test public void testApplySdkEnvironmentOverridesByDefault() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); - String dockerHubPythonContainerUrl = "apache/beam_python3.10_sdk:latest"; - String gcrPythonContainerUrl = "gcr.io/cloud-dataflow/v1beta3/beam_python3.10_sdk:latest"; + String dockerHubPythonContainerUrl = "apache/beam_python3.11_sdk:latest"; + String gcrPythonContainerUrl = "gcr.io/cloud-dataflow/v1beta3/beam_python3.11_sdk:latest"; DataflowRunner runner = DataflowRunner.fromOptions(options); RunnerApi.Pipeline pipeline = containerUrlToPipeline(dockerHubPythonContainerUrl); RunnerApi.Pipeline expectedPipeline = containerUrlToPipeline(gcrPythonContainerUrl); @@ -1303,8 +1303,8 @@ public void testApplySdkEnvironmentOverridesByDefault() throws IOException { @Test public void testApplySdkEnvironmentOverridesRcByDefault() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); - String dockerHubPythonContainerUrl = "apache/beam_python3.10_sdk:2.68.0rc2"; - String gcrPythonContainerUrl = "gcr.io/cloud-dataflow/v1beta3/beam_python3.10_sdk:2.68.0"; + String dockerHubPythonContainerUrl = "apache/beam_python3.11_sdk:2.68.0rc2"; + String gcrPythonContainerUrl = "gcr.io/cloud-dataflow/v1beta3/beam_python3.11_sdk:2.68.0"; DataflowRunner runner = DataflowRunner.fromOptions(options); RunnerApi.Pipeline pipeline = containerUrlToPipeline(dockerHubPythonContainerUrl); RunnerApi.Pipeline expectedPipeline = containerUrlToPipeline(gcrPythonContainerUrl); diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 0e2acbe46748..a64bb805c96a 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -70,7 +70,7 @@ import warnings if sys.version_info.major == 3: - if sys.version_info.minor <= 9 or sys.version_info.minor >= 15: + if sys.version_info.minor <= 10 or sys.version_info.minor >= 15: warnings.warn( 'This version of Apache Beam has not been sufficiently tested on ' 'Python %s.%s. You may encounter bugs or missing features.' % diff --git a/sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/Dockerfile b/sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/Dockerfile index a62b9edd4060..a4ebf7d1d472 100644 --- a/sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/Dockerfile +++ b/sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/Dockerfile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.10-slim +FROM python:3.11-slim WORKDIR /app RUN pip install flask gunicorn COPY echo_server.py main.py diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 0875bdd14df5..654808373f24 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -87,9 +87,7 @@ _LOGGER = logging.getLogger(__name__) -_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = [ - '3.10', '3.11', '3.12', '3.13', '3.14' -] +_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.11', '3.12', '3.13', '3.14'] class Environment(object): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 4fca13abee99..fdce35b4b5ef 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -1119,7 +1119,7 @@ def test_interpreter_version_check_passes_with_experiment(self): 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') - def test_interpreter_version_check_fails_py38(self): + def test_interpreter_version_check_fails_py310(self): pipeline_options = PipelineOptions([]) self.assertRaises( Exception, @@ -1128,12 +1128,12 @@ def test_interpreter_version_check_fails_py38(self): @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info', - (3, 10, 10)) + (3, 11, 4)) @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', '2.2.0') - def test_interpreter_version_check_passes_py310(self): + def test_interpreter_version_check_passes_py311(self): pipeline_options = PipelineOptions([]) apiclient._verify_interpreter_version_is_supported(pipeline_options) diff --git a/sdks/python/apache_beam/testing/benchmarks/README.md b/sdks/python/apache_beam/testing/benchmarks/README.md index f91c2120e7f6..2f3d260c3b96 100644 --- a/sdks/python/apache_beam/testing/benchmarks/README.md +++ b/sdks/python/apache_beam/testing/benchmarks/README.md @@ -116,7 +116,7 @@ Navigate to [`.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml`](../../ arguments: | -PloadTest.mainClass=apache_beam.testing.benchmarks.wordcount.wordcount \ -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ + -PpythonVersion=3.11 \ '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-wordcount-python-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_wordcount-${{env.NOW_UTC}}.txt' \ ``` diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index de9488afee98..c10739f2f831 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -233,7 +233,7 @@ platform_identifiers_map.each { platform, idsuffix -> } getVersionsAsList('python_versions').each { it -> def pyversion = it.replace('.', '') - def cibuildwheel_version = it == '3.10' ? '2.23.3' : '3.4.0' + def cibuildwheel_version = '3.4.0' project.tasks.register("bdistPy${pyversion}${platform}") { description "Build a Python wheel distribution for Py${pyversion} ${platform}" diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index 8c6dd74a2f91..44e27672e7cf 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -87,7 +87,7 @@ RUN \ # Update ensurepip to use most recent versions of setuptools and pip. This avoids some vulnerabilities which won't be fixed on older versions of python. # upgrade_ensurepip is not maintained for Python 3.12+ (ensurepip no longer bundles setuptools). - if [ "${py_version}" = "3.10" ] || [ "${py_version}" = "3.11" ]; then \ + if [ "${py_version}" = "3.11" ]; then \ pip install upgrade_ensurepip; \ python3 -m upgrade_ensurepip; \ find /usr/local/lib/python${py_version}/ensurepip/_bundled/setuptools-* -type f ! -name $(basename $(ls -v /usr/local/lib/python${py_version}/ensurepip/_bundled/setuptools-*-py3-none-any.whl | tail -n 1)) -delete; \ diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index ed1f57faac4e..ab6fbdc94094 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -31,8 +31,8 @@ # https://s.apache.org/beam-python-dev-wiki if [[ $# -lt 2 ]]; then - printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.10 " - printf "\n\where 3.10 is the Python major.minor version." + printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.11 " + printf "\n\where 3.11 is the Python major.minor version." exit 1 fi diff --git a/sdks/python/expansion-service-container/Dockerfile b/sdks/python/expansion-service-container/Dockerfile index 770ac7f9fbef..b0150fba04f8 100644 --- a/sdks/python/expansion-service-container/Dockerfile +++ b/sdks/python/expansion-service-container/Dockerfile @@ -17,8 +17,8 @@ ############################################################################### # We just need to support one Python version supported by Beam. -# Picking the current default Beam Python version which is Python 3.10. -ARG py_version=3.10 +# Picking the current default Beam Python version which is Python 3.11. +ARG py_version=3.11 FROM python:${py_version}-bookworm as expansion-service LABEL Author "Apache Beam " ARG TARGETOS diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 75a6b4fc6ed6..f2abee2e9cbb 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -84,7 +84,7 @@ exclude = [ "**/portability/api/**/*.py", "**/portability/api/__init__.py", ] -target-version = "py310" +target-version = "py311" src = ["apache_beam"] line-length = 120 @@ -179,7 +179,7 @@ known-first-party = ["apache_beam"] [tool.pyrefly] project-includes = ["apache_beam"] -python-version = "3.10.0" +python-version = "3.11.0" ignore-missing-imports = ["*"] untyped-def-behavior = "skip-and-infer-return-any" diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 97ba1d304926..e1aeb43a0b62 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -335,7 +335,7 @@ def get_portability_package_data(): return files -python_requires = '>=3.10' +python_requires = '>=3.11' if sys.version_info.major == 3 and sys.version_info.minor >= 15: warnings.warn( @@ -608,6 +608,9 @@ def get_portability_package_data(): 'p310_ml_test': [ 'datatable', ] + ml_base + qdrant_dependency, + 'p311_ml_test': [ + 'datatable', + ] + ml_base + qdrant_dependency, 'p312_ml_test': [ 'datatable', ] + ml_base + qdrant_dependency, @@ -690,7 +693,6 @@ def get_portability_package_data(): 'Intended Audience :: End Users/Desktop', 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 7e6919d18ab8..484710fc785c 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -669,7 +669,7 @@ project.tasks.register("inferencePostCommitIT") { } // The base NVIDIA image used for TensorRT contains Python 3.12, so the TensorRT -// suite belongs here rather than in the py310 suite above. +// suite belongs here rather than in the py311 suite above. project.tasks.register("inferencePostCommitITPy312") { dependsOn = [ 'tensorRTtests', diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index c9910b801629..8370d3e6f9a0 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -22,14 +22,14 @@ # dataflow test-suites # (TODO): https://github.com/apache/beam/issues/21971 -# Add python 3.10 to dataflow test-suites -dataflow_precommit_it_task_py_versions=3.10,3.14 -dataflow_mongodbio_it_task_py_versions=3.10 -dataflow_chicago_taxi_example_task_py_versions=3.10 +# Add python 3.11 to dataflow test-suites +dataflow_precommit_it_task_py_versions=3.11,3.14 +dataflow_mongodbio_it_task_py_versions=3.11 +dataflow_chicago_taxi_example_task_py_versions=3.11 # TODO: Enable following tests after making sure we have enough capacity. -dataflow_validates_runner_batch_tests=3.10,3.14 -dataflow_validates_runner_streaming_tests=3.10,3.14 +dataflow_validates_runner_batch_tests=3.11,3.14 +dataflow_validates_runner_streaming_tests=3.11,3.14 dataflow_examples_postcommit_py_versions=3.14 # Run cloudml TFT benchmark tests on Python 3.11 to avoid Python 3.10 protobuf<=6.32 cap. dataflow_cloudml_benchmark_tests_py_versions=3.11 @@ -38,16 +38,16 @@ direct_mongodbio_it_task_py_versions=3.14 # flink runner test-suites flink_validates_runner_precommit_py_versions=3.14 -flink_validates_runner_postcommit_py_versions=3.10,3.14 -flink_examples_postcommit_py_versions=3.10,3.14 +flink_validates_runner_postcommit_py_versions=3.11,3.14 +flink_examples_postcommit_py_versions=3.11,3.14 # spark runner test-suites -spark_examples_postcommit_py_versions=3.10,3.14 +spark_examples_postcommit_py_versions=3.11,3.14 # prism runner test-suites prism_validates_runner_precommit_py_versions=3.14 -prism_validates_runner_postcommit_py_versions=3.10,3.14 -prism_examples_postcommit_py_versions=3.10,3.14 +prism_validates_runner_postcommit_py_versions=3.11,3.14 +prism_examples_postcommit_py_versions=3.11,3.14 # cross language postcommit python test suites -cross_language_validates_py_versions=3.10,3.14 +cross_language_validates_py_versions=3.11,3.14 diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index fabf9fd4365a..a2b0f0d5e235 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -26,5 +26,191 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.11' +def posargs = project.findProperty("posargs") ?: "" + apply from: "../common.gradle" +toxTask "testPy311CloudCoverage", "py311-cloudcoverage", "${posargs}" +test.dependsOn "testPy311CloudCoverage" +project.tasks.register("preCommitPyCoverage") { + dependsOn = ["testPy311CloudCoverage"] +} + +// Dep Postcommit runs test suites that evaluate compatibility of particular +// dependencies. Each suite is exercised on at most one python version. +// +// Should still leave at least one version in PreCommit unless the marked tests +// are also exercised by existing PreCommit +// e.g. pyarrow and pandas also run on PreCommit Dataframe and Coverage +project.tasks.register("postCommitPyDep") {} + +// Create a test task for supported major versions of pyarrow. +// Keep in sync with [testenv:py{310,311}-pyarrow-...] in tox.ini +// (versions released in roughly the last 1-2 years). + +toxTask "testPy311pyarrow-17", "py311-pyarrow-17", "${posargs}" +test.dependsOn "testPy311pyarrow-17" +postCommitPyDep.dependsOn "testPy311pyarrow-17" + +toxTask "testPy311pyarrow-18", "py311-pyarrow-18", "${posargs}" +test.dependsOn "testPy311pyarrow-18" +postCommitPyDep.dependsOn "testPy311pyarrow-18" + +toxTask "testPy311pyarrow-19", "py311-pyarrow-19", "${posargs}" +test.dependsOn "testPy311pyarrow-19" +postCommitPyDep.dependsOn "testPy311pyarrow-19" + +toxTask "testPy311pyarrow-20", "py311-pyarrow-20", "${posargs}" +test.dependsOn "testPy311pyarrow-20" +postCommitPyDep.dependsOn "testPy311pyarrow-20" + +toxTask "testPy311pyarrow-21", "py311-pyarrow-21", "${posargs}" +test.dependsOn "testPy311pyarrow-21" +postCommitPyDep.dependsOn "testPy311pyarrow-21" + +toxTask "testPy311pyarrow-22", "py311-pyarrow-22", "${posargs}" +test.dependsOn "testPy311pyarrow-22" +postCommitPyDep.dependsOn "testPy311pyarrow-22" + +toxTask "testPy311pyarrow-23", "py311-pyarrow-23", "${posargs}" +test.dependsOn "testPy311pyarrow-23" +postCommitPyDep.dependsOn "testPy311pyarrow-23" + +toxTask "testPy311pyarrow-24", "py311-pyarrow-24", "${posargs}" +test.dependsOn "testPy311pyarrow-24" +postCommitPyDep.dependsOn "testPy311pyarrow-24" + +toxTask "testPy311pyarrow-25", "py311-pyarrow-25", "${posargs}" +test.dependsOn "testPy311pyarrow-25" +postCommitPyDep.dependsOn "testPy311pyarrow-25" + +// Create a test task for each supported minor version of pandas +toxTask "testPy311pandas-14", "py311-pandas-14", "${posargs}" +test.dependsOn "testPy311pandas-14" +postCommitPyDep.dependsOn "testPy311pandas-14" + +toxTask "testPy311pandas-15", "py311-pandas-15", "${posargs}" +test.dependsOn "testPy311pandas-15" +postCommitPyDep.dependsOn "testPy311pandas-15" + +toxTask "testPy311pandas-20", "py311-pandas-20", "${posargs}" +test.dependsOn "testPy311pandas-20" +postCommitPyDep.dependsOn "testPy311pandas-20" + +// TODO(https://github.com/apache/beam/issues/31192): Add below suites +// after dependency compat tests suite switches to Python 3.9 or we add +// Python 2.2 support. + +// toxTask "testPy311pandas-21", "py311-pandas-21", "${posargs}" +// test.dependsOn "testPy311pandas-21" +// postCommitPyDep.dependsOn "testPy311pandas-21" + +// toxTask "testPy311pandas-22", "py311-pandas-22", "${posargs}" +// test.dependsOn "testPy311pandas-22" +// postCommitPyDep.dependsOn "testPy311pandas-22" + +// TODO(https://github.com/apache/beam/issues/30908): Revise what are we testing + +// Create a test task for each minor version of pytorch + +toxTask "testPy311pytorch-111", "py311-pytorch-111", "${posargs}" +test.dependsOn "testPy311pytorch-111" +postCommitPyDep.dependsOn "testPy311pytorch-111" + +toxTask "testPy311pytorch-112", "py311-pytorch-112", "${posargs}" +test.dependsOn "testPy311pytorch-112" +postCommitPyDep.dependsOn "testPy311pytorch-112" + +toxTask "testPy311pytorch-113", "py311-pytorch-113", "${posargs}" +test.dependsOn "testPy311pytorch-113" +postCommitPyDep.dependsOn "testPy311pytorch-113" + +// run on precommit +toxTask "testPy311pytorch-200", "py311-pytorch-200", "${posargs}" +test.dependsOn "testPy311pytorch-200" +postCommitPyDep.dependsOn "testPy311pytorch-200" + +toxTask "testPy311tft-113", "py311-tft-113", "${posargs}" +test.dependsOn "testPy311tft-113" +postCommitPyDep.dependsOn "testPy311tft-113" + +// TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task once onnx supports protobuf 4.x.x +// Create a test task for each minor version of onnx +// toxTask "testPy311onnx-113", "py311-onnx-113", "${posargs}" +// test.dependsOn "testPy311onnx-113" +// postCommitPyDep.dependsOn "testPy311onnx-113" + +// Create a test task for each minor version of tensorflow +toxTask "testPy311tensorflow-212", "py311-tensorflow-212", "${posargs}" +test.dependsOn "testPy311tensorflow-212" +postCommitPyDep.dependsOn "testPy311tensorflow-212" + +// Create a test task for each minor version of transformers +toxTask "testPy311transformers-428", "py311-transformers-428", "${posargs}" +test.dependsOn "testPy311transformers-428" +postCommitPyDep.dependsOn "testPy311transformers-428" + +toxTask "testPy311transformers-447", "py311-transformers-447", "${posargs}" +test.dependsOn "testPy311transformers-447" +postCommitPyDep.dependsOn "testPy311transformers-447" + +toxTask "testPy311transformers-latest", "py311-transformers-latest", "${posargs}" +test.dependsOn "testPy311transformers-latest" +postCommitPyDep.dependsOn "testPy311transformers-latest" + +toxTask "testPy311embeddingsMLTransform", "py311-embeddings", "${posargs}" +test.dependsOn "testPy311embeddingsMLTransform" +postCommitPyDep.dependsOn "testPy311embeddingsMLTransform" + +// Part of MLTransform embeddings test suite but requires tensorflow hub, which we need to test on +// mutliple versions so keeping this suite separate. +toxTask "testPy311TensorflowHubEmbeddings-014", "py311-TFHubEmbeddings-014", "${posargs}" +test.dependsOn "testPy311TensorflowHubEmbeddings-014" +postCommitPyDep.dependsOn "testPy311TensorflowHubEmbeddings-014" + +toxTask "testPy311TensorflowHubEmbeddings-015", "py311-TFHubEmbeddings-015", "${posargs}" +test.dependsOn "testPy311TensorflowHubEmbeddings-015" +postCommitPyDep.dependsOn "testPy311TensorflowHubEmbeddings-015" + +toxTask "whitespacelint", "whitespacelint", "${posargs}" + +task archiveFilesToLint(type: Zip) { + archiveFileName = "files-to-whitespacelint.zip" + destinationDirectory = file("$buildDir/dist") + + from ("$rootProject.projectDir") { + include "**/*.md" + include "**/build.gradle" + include '**/build.gradle.kts' + exclude '**/build/**' // intermediate build directory + exclude 'website/www/site/themes/docsy/**' // fork to google/docsy + exclude "**/node_modules/*" + exclude "**/.gogradle/*" + } +} + +task unpackFilesToLint(type: Copy) { + from zipTree("$buildDir/dist/files-to-whitespacelint.zip") + into "$buildDir/files-to-whitespacelint" +} + +whitespacelint.dependsOn archiveFilesToLint, unpackFilesToLint +unpackFilesToLint.dependsOn archiveFilesToLint +archiveFilesToLint.dependsOn cleanPython + +toxTask "jest", "jest", "${posargs}" + +toxTask "eslint", "eslint", "${posargs}" + +task copyTsSource(type: Copy) { + from ("$rootProject.projectDir") { + include "sdks/python/apache_beam/runners/interactive/extensions/**/*" + exclude "sdks/python/apache_beam/runners/interactive/extensions/**/lib/*" + exclude "sdks/python/apache_beam/runners/interactive/extensions/**/node_modules/*" + } + into "$buildDir/ts" +} + +jest.dependsOn copyTsSource +eslint.dependsOn copyTsSource +copyTsSource.dependsOn cleanPython diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 8f59766612cd..c88e5e2fea6a 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py310,py311,py312,py313,py314,py310-{cloud,cloudcoverage,dask},py311-{cloud,dask},py312-{cloud,dask},py313-{cloud,dask},py314-{cloud,dask},docs,lint,whitespacelint +envlist = py311,py312,py313,py314,py311-{cloud,cloudcoverage,dask},py312-{cloud,dask},py313-{cloud,dask},py314-{cloud,dask},docs,lint,whitespacelint toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] @@ -67,7 +67,7 @@ commands_post = commands = false {envname} is misconfigured -[testenv:py{310,311,312,313,314}] +[testenv:py{311,312,313,314}] commands_pre = python --version pip --version @@ -79,7 +79,7 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{310,311,312,313,314}-macos] +[testenv:py{311,312,313,314}-macos] commands_pre = python --version pip --version @@ -89,28 +89,28 @@ commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{310,311,312,313,314}-win] +[testenv:py{311,312,313,314}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py{310,311,312,313,314}-cloud] +[testenv:py{311,312,313,314}-cloud] ; extras = test,gcp,interactive,dataframe,aws,azure extras = test,hadoop,gcp,interactive,dataframe,aws,azure commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{310,311}-ml] +[testenv:py311-ml] # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. deps = pip==25.0.1 accelerate>=1.6.0 onnx<1.19.0 setenv = -extras = test,gcp,dataframe,p310_ml_test +extras = test,gcp,dataframe,p311_ml_test commands = # Log tensorflow version for debugging /bin/sh -c "pip freeze | grep -E tensorflow" @@ -150,14 +150,14 @@ commands = /bin/sh -c "pip freeze | grep -E tensorflow" bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{310,311,312,313,314}-dask] +[testenv:py{311,312,313,314}-dask] extras = test,dask,dataframe commands_pre = pip install 'distributed>=2024.4.2' 'dask>=2024.4.2' commands = bash {toxinidir}/scripts/run_pytest.sh {envname} {toxinidir}/apache_beam/runners/dask/ -[testenv:py{310,311,312,313,314}-win-dask] +[testenv:py{311,312,313,314}-win-dask] # use the tight range since the latest dask requires cloudpickle 3.0 commands_pre = pip install 'distributed>=2024.4.2,<2024.9.0' 'dask>=2024.4.2,<2024.9.0' @@ -167,7 +167,7 @@ commands = install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py310-cloudcoverage] +[testenv:py311-cloudcoverage] deps = pytest-cov==3.0.0 @@ -340,7 +340,7 @@ extras = test commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/prism_runner_test.py {posargs} -[testenv:py{310,311}-pyarrow-{17,18,19,20,21,22,23,24,25}] +[testenv:py311-pyarrow-{17,18,19,20,21,22,23,24,25}] deps = # Test against versions of pyarrow released in last ~2 years. 17: pyarrow>=17,<18 @@ -362,7 +362,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{310,311}-pandas-{14,15,20}] +[testenv:py311-pandas-{14,15,20}] deps = 14: pandas>=1.4.3,<1.5.0 14: numpy>=1.14.3,<1.27.0 @@ -378,7 +378,7 @@ commands = # Run all DataFrame API unit tests bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe' -[testenv:py{310,311}-tft-{113,114}] +[testenv:py311-tft-{113,114}] deps = # Help pip resolve conflict with typing-extensions due to an old version of tensorflow https://github.com/apache/beam/issues/30852 113: pydantic<2.0 @@ -386,7 +386,7 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py{310,311}-pytorch-{19,110,111,112,113}] +[testenv:py311-pytorch-{19,110,111,112,113}] deps = 19: torch>=1.9.0,<1.10.0 110: torch>=1.10.0,<1.11.0 @@ -404,7 +404,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{310,311}-pytorch-200] +[testenv:py311-pytorch-200] deps = 200: torch>=2.0.0,<2.1.0 @@ -436,8 +436,8 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -# TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py310/build.gradle once onnx supports protobuf 4.x.x -[testenv:py{310,311}-onnx-113] +# TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py311/build.gradle once onnx supports protobuf 4.x.x +[testenv:py311-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...} # when multiple onnx versions are tested. @@ -456,7 +456,7 @@ commands = # Run all ONNX unit tests pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs} -[testenv:py310-tensorflow-212] +[testenv:py311-tensorflow-212] deps = 212: googleapis-common-protos<1.70 @@ -489,7 +489,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py310-xgboost-{160,170}] +[testenv:py311-xgboost-{160,170}] deps = 160: xgboost>=1.6.0,<1.7.0 @@ -505,7 +505,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{310,311}-transformers-{428,447,latest}] +[testenv:py311-transformers-{428,447,latest}] deps = # Environment dependencies are defined in the `setenv` section and installed in the `commands` section. pip_pre = False @@ -526,7 +526,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_transformers {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{310,313}-vertex-ai] +[testenv:py{311,313}-vertex-ai] deps = tensorflow==2.12.0 extras = test,gcp @@ -539,7 +539,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{310,311}-embeddings] +[testenv:py311-embeddings] deps = sentence-transformers==3.3.1 accelerate>=1.6.0 @@ -557,7 +557,7 @@ commands = /bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{310,311}-TFHubEmbeddings-{014,015}] +[testenv:py311-TFHubEmbeddings-{014,015}] deps = 014: tensorflow-hub>=0.14.0,<0.15.0 # Help pip resolve conflict with typing-extensions due to an old version of tensorboard https://github.com/apache/beam/issues/30852 @@ -574,7 +574,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms/embeddings' -[testenv:py{310,312}-dill] +[testenv:py{311,312}-dill] extras = test,dill commands = # Log dill version for debugging diff --git a/website/www/site/content/en/documentation/runtime/environments.md b/website/www/site/content/en/documentation/runtime/environments.md index 3ebabf85385d..049fdb9da9f7 100644 --- a/website/www/site/content/en/documentation/runtime/environments.md +++ b/website/www/site/content/en/documentation/runtime/environments.md @@ -117,10 +117,10 @@ This method requires building image artifacts from Beam source. For additional i ./gradlew :sdks:java:container:java17:docker ./gradlew :sdks:java:container:java21:docker ./gradlew :sdks:go:container:docker - ./gradlew :sdks:python:container:py310:docker ./gradlew :sdks:python:container:py311:docker ./gradlew :sdks:python:container:py312:docker ./gradlew :sdks:python:container:py313:docker + ./gradlew :sdks:python:container:py314:docker # Shortcut for building all Python SDKs ./gradlew :sdks:python:container:buildAll @@ -134,10 +134,10 @@ This method requires building image artifacts from Beam source. For additional i apache/beam_java11_sdk latest sha256:... ... 1 min ago ... apache/beam_java17_sdk latest sha256:... ... 1 min ago ... apache/beam_java21_sdk latest sha256:... ... 1 min ago ... - apache/beam_python3.10_sdk latest sha256:... ... 1 min ago ... apache/beam_python3.11_sdk latest sha256:... ... 1 min ago ... apache/beam_python3.12_sdk latest sha256:... ... 1 min ago ... apache/beam_python3.13_sdk latest sha256:... ... 1 min ago ... + apache/beam_python3.14_sdk latest sha256:... ... 1 min ago ... apache/beam_go_sdk latest sha256:... ... 1 min ago ... ``` diff --git a/website/www/site/content/en/documentation/sdks/python-sdk-image-build.md b/website/www/site/content/en/documentation/sdks/python-sdk-image-build.md index f456a686afea..e63d15760c7d 100644 --- a/website/www/site/content/en/documentation/sdks/python-sdk-image-build.md +++ b/website/www/site/content/en/documentation/sdks/python-sdk-image-build.md @@ -52,7 +52,7 @@ docker push us-central1-docker.pkg.dev///beam_python3 You can build a docker image if your local environment has Java, Python, Golang and Docker installation. Try `./gradlew :sdks:python:container:py:docker`. For example, -`:sdks:python:container:py310:docker` builds `apache/beam_python3.10_sdk` +`:sdks:python:container:py311:docker` builds `apache/beam_python3.11_sdk` locally if successful. You can follow this guide building a custom image from a VM if the build fails in your local environment. @@ -249,10 +249,10 @@ Run the Gradle task to start Docker image build. This will take several minutes. You can run `:sdks:python:container:py:docker` to build an image for different Python version. See [the supported Python version list](https://github.com/apache/beam/tree/master/sdks/python/container). -For example, `py310` is for Python 3.10. +For example, `py311` is for Python 3.11. ```shell -./gradlew :sdks:python:container:py310:docker +./gradlew :sdks:python:container:py311:docker ``` If the build is successful, you can see the built image locally.