From 5d97388454cb47a2db08c23e6847c644e55ada01 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 12 Oct 2026 16:24:20 -0400 Subject: [PATCH] Clean up Python load test version configs * Avoid time.sleep(0) invocation in Synthetic Source that causing regression in Py311 * Remove no-op "--experiments=use_runner_v2" * Use snapshot SDK containers for load test, instead of default to beam-master --- ...marks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt | 1 - ...marks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt | 1 - ...Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt | 1 - ..._Python_Benchmarks_Dataflow_Pytorch_Image_Captioning.txt | 1 - ...n_Benchmarks_Dataflow_Pytorch_Image_Object_Detection.txt | 1 - ...Python_Benchmarks_Dataflow_Table_Row_Inference_Batch.txt | 1 - ...ython_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt | 1 - ...nference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt | 1 - .../python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt | 1 - .../python_CoGBK_Flink_Batch_100b_Single_Key.txt | 1 - .../python_CoGBK_Flink_Batch_10kB.txt | 1 - ...ython_Combine_Dataflow_Streaming_2GB_10_byte_records.txt | 1 - .../python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt | 1 - .../python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt | 1 - .../python_Combine_Flink_Batch_2GB_10_byte_records.txt | 1 - .../python_Combine_Flink_Batch_2GB_Fanout_4.txt | 1 - .../python_Combine_Flink_Batch_2GB_Fanout_8.txt | 1 - .../python_Combine_Flink_Streaming_2GB_10_byte_records.txt | 1 - .../python_Combine_Flink_Streaming_2GB_Fanout_4.txt | 1 - .../python_Combine_Flink_Streaming_2GB_Fanout_8.txt | 1 - .../python_Combine_Flink_Streaming_small_Fanout_1.txt | 1 - .../python_Combine_Flink_Streaming_small_Fanout_2.txt | 1 - .../python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt | 1 - .../python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt | 1 - .../python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt | 1 - ...eaming_fanout_4_times_with_2GB_10-byte_records_total.txt | 1 - ...eaming_fanout_8_times_with_2GB_10-byte_records_total.txt | 1 - .../python_GBK_Flink_Batch_2GB_of_100B_records.txt | 1 - .../python_GBK_Flink_Batch_2GB_of_10B_records.txt | 1 - ..._Batch_fanout_4_times_with_2GB_10-byte_records_total.txt | 1 - ..._Batch_fanout_8_times_with_2GB_10-byte_records_total.txt | 1 - ...python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt | 1 - ...ate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt | 1 - ...rate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt | 1 - .../python_ParDo_Dataflow_Streaming_100_Counters.txt | 1 - .../python_ParDo_Dataflow_Streaming_10_Counters.txt | 1 - .../python_ParDo_Dataflow_Streaming_10_Iterations.txt | 1 - .../python_ParDo_Dataflow_Streaming_200_Iterations.txt | 1 - .../python_ParDo_Flink_Batch_10_Counters.txt | 1 - .../python_ParDo_Flink_Batch_10_Iterations.txt | 1 - .../python_ParDo_Flink_Batch_200_Iterations.txt | 1 - .../python_ParDo_Flink_Streaming_100_Counters.txt | 1 - .../python_ParDo_Flink_Streaming_10_Counters.txt | 1 - .../python_ParDo_Flink_Streaming_10_Iterations.txt | 1 - .../python_ParDo_Flink_Streaming_200_Iterations.txt | 1 - .../python_ParDo_Flink_Streaming_5_Iterations.txt | 1 - ...eInput_Dataflow_Batch_10gb_1000window_first_iterable.txt | 1 - ...on_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt | 1 - ...SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt | 1 - ...ython_SideInput_Dataflow_Batch_10gb_1window_iterable.txt | 1 - ...nput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt | 1 - ...put_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt | 1 - ...deInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt | 1 - ...eInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt | 1 - ...thon_SideInput_Dataflow_Batch_1gb_1window_first_list.txt | 1 - .../python_SideInput_Dataflow_Batch_1gb_1window_list.txt | 1 - sdks/python/apache_beam/testing/load_tests/build.gradle | 5 +++++ sdks/python/apache_beam/testing/synthetic_pipeline.py | 6 ++++-- 58 files changed, 9 insertions(+), 58 deletions(-) diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt index 37c47ca25542..eec6802e107b 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt @@ -38,5 +38,4 @@ --max_batch_size=64 --embedding_min_ram=16GB --dataflow_service_options=enable_prime ---experiments=use_runner_v2 --timeout_ms=3600000 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt index 274dc43f9416..464b59a7c734 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt @@ -40,5 +40,4 @@ --sdk_location=container --sdk_container_image=us.gcr.io/apache-beam-testing/python-postcommit-it/tensor_rt:latest --dataflow_service_options=enable_prime ---experiments=use_runner_v2 --timeout_ms=3600000 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt index 1835dc260025..172a924934d1 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt @@ -39,4 +39,3 @@ --device=CPU --sdk_location=container --sdk_container_image=us.gcr.io/apache-beam-testing/python-postcommit-it/tensor_rt:latest ---experiments=use_runner_v2 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Captioning.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Captioning.txt index c58707383ef9..329235dad2cf 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Captioning.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Captioning.txt @@ -34,4 +34,3 @@ --clip_batch_size=8 --clip_score_normalize=false --runner=DataflowRunner ---experiments=use_runner_v2 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Object_Detection.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Object_Detection.txt index 18f017ad0109..4ba0fb59c4bd 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Object_Detection.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Image_Object_Detection.txt @@ -34,4 +34,3 @@ --input=gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt --model_state_dict_path=gs://apache-beam-ml/models/torchvision.detection.fasterrcnn_resnet50_fpn.pth --runner=DataflowRunner ---experiments=use_runner_v2 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Batch.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Batch.txt index 36b3527dcbb2..038968a1b58c 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Batch.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Batch.txt @@ -37,4 +37,3 @@ --feature_columns=feature1,feature2,feature3,feature4,feature5 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_batch_outputs --runner=DataflowRunner ---experiments=use_runner_v2 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt index 39ce9071840b..d08aa08d346f 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt @@ -39,4 +39,3 @@ --feature_columns=feature1,feature2,feature3,feature4,feature5 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_stream_outputs --runner=DataflowRunner ---experiments=use_runner_v2 diff --git a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt index fd2101afa3f1..fa3de4660315 100644 --- a/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt +++ b/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt @@ -34,4 +34,3 @@ --model_gcs_path=gs://apache-beam-ml/models/gemma-2b-it --requirements_file=apache_beam/ml/inference/vllm_tests_requirements.txt --dataflow_service_options=worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver ---experiments=use_runner_v2 diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt index 2b611fc8bf72..34c820e9a2d6 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt @@ -25,4 +25,3 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt index 4d66b54e1fc2..0b34b0ecc999 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt @@ -25,4 +25,3 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt index fd490d8aec1f..af2f579913cb 100644 --- a/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt +++ b/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt @@ -25,4 +25,3 @@ --runner=PortableRunner --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt index 684d9cb85ec3..2ecb3ffba648 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt @@ -25,5 +25,4 @@ --autoscaling_algorithm=NONE --top_count=20 --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt index 03b26b9d4e13..f34c3bdd60ee 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt @@ -26,5 +26,4 @@ --fanout=4 --top_count=20 --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt index 46d68261342f..6290652f4fff 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt @@ -26,5 +26,4 @@ --fanout=8 --top_count=20 --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt index a66514b46cd1..52671790849e 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_10_byte_records.txt @@ -22,6 +22,5 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt index 70c91c576136..6257f550ec71 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_4.txt @@ -22,7 +22,6 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=4 --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt index 266890b7e328..2d8014c708cc 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Batch_2GB_Fanout_8.txt @@ -22,7 +22,6 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=8 --top_count=20 --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt index 02dab88ec24b..19d06df9da77 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_10_byte_records.txt @@ -22,7 +22,6 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --top_count=20 --streaming --use_stateful_load_generator diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt index 8861a7512663..1d10875e869d 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt @@ -22,7 +22,6 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=4 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt index d713b4b8a2fb..362230a62ad0 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt @@ -22,7 +22,6 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=8 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt index 9b89c8a97322..a62c9c2c0835 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt @@ -22,7 +22,6 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=1 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt index b12f966b7323..bce2e973f5cd 100644 --- a/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt +++ b/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt @@ -22,7 +22,6 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --fanout=2 --top_count=20 --streaming diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt index 057f71d5627c..f31f59be61f4 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt @@ -26,5 +26,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt index 57c1be11d592..cfd90266e5d8 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt @@ -26,5 +26,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt index 64d224a4663f..87aef8f8229b 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt @@ -26,5 +26,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt index 8e38713cc66d..297ebde6dfbd 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -26,5 +26,4 @@ --num_workers=16 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt index 35508480662c..81def535ea68 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -26,5 +26,4 @@ --num_workers=16 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt index 11e14ffb5763..0f71cc3fc517 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_100B_records.txt @@ -24,5 +24,4 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt index 6fb19aa6e6d6..92d432d660fe 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_2GB_of_10B_records.txt @@ -24,5 +24,4 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt index bf0a17978a76..c03b60f3a1e0 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,4 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt index f76d6176b894..53ab970ac9df 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -24,5 +24,4 @@ --parallelism=16 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt index 1154da900faf..96d9c023281e 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt @@ -24,5 +24,4 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt index 6cb1e68aeafc..6d2df1b63d95 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt @@ -26,5 +26,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt index 712749090aec..81d6d8fbed94 100644 --- a/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt +++ b/.github/workflows/load-tests-pipeline-options/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt @@ -26,5 +26,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt index 71fc818d0e07..92f2448f6e7c 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_100_Counters.txt @@ -27,5 +27,4 @@ --num_****s=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt index fe48dedfa837..3666ab866777 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Counters.txt @@ -27,5 +27,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt index 84d2cdb7a85b..268ab0c119e6 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_10_Iterations.txt @@ -27,5 +27,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt index 02636335cb37..55f012f3068f 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Dataflow_Streaming_200_Iterations.txt @@ -27,5 +27,4 @@ --num_workers=5 --autoscaling_algorithm=NONE --streaming ---experiments=use_runner_v2 --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt index 8ae1401bc2c7..f1acdb7a2787 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Counters.txt @@ -25,5 +25,4 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt index d822b109ca96..3ee60e139e2a 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_10_Iterations.txt @@ -25,5 +25,4 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt index 8ae1401bc2c7..f1acdb7a2787 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Batch_200_Iterations.txt @@ -25,5 +25,4 @@ --parallelism=5 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt index d1386a9ce3e9..92627146c999 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_100_Counters.txt @@ -26,6 +26,5 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt index c54710fc6eb2..fef8a9cc443b 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Counters.txt @@ -26,6 +26,5 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt index de0a77fb62af..3800cd3752ad 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_10_Iterations.txt @@ -27,6 +27,5 @@ --stateful --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt index 6071a2b7404a..9f0b6fa0a100 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_200_Iterations.txt @@ -26,6 +26,5 @@ --streaming --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt index d306b8573de5..27ac1d2640a0 100644 --- a/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt +++ b/.github/workflows/load-tests-pipeline-options/python_ParDo_Flink_Streaming_5_Iterations.txt @@ -30,6 +30,5 @@ --shutdown_sources_after_idle_ms=300000 --job_endpoint=localhost:8099 --environment_type=DOCKER ---environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.11_sdk:latest --use_stateful_load_generator --runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt index 204c07bc16a2..99cf392768ea 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_9 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=iter --access_percentage=1 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt index 1ae64bb4a369..df5e1d27872f 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_10 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=iter --window_count=1000 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt index 0759517d9c2e..8e0548364f73 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_3 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=iter --access_percentage=1 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt index c555c0d32d4c..95b552b48eb1 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_4 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=iter --runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt index 4b3cee817f43..ef06616a853d 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_7 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=dict --access_percentage=1 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt index 00ba6feef50c..22a18b887ea8 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_8 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=dict --access_percentage=99 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt index 07e4a5ecba62..06b5a006ddc4 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_1 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=dict --access_percentage=1 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt index b565598c1a16..16e4a38c9d60 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_2 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=dict --access_percentage=99 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt index de5b02198177..8acd7c6d8220 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_5 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=list --access_percentage=1 diff --git a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt index 078ce2cc74fe..9eb8fce60480 100644 --- a/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt +++ b/.github/workflows/load-tests-pipeline-options/python_SideInput_Dataflow_Batch_1gb_1window_list.txt @@ -22,7 +22,6 @@ --influx_measurement=python_batch_sideinput_6 --num_workers=10 --autoscaling_algorithm=NONE ---experiments=use_runner_v2 --input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' --side_input_type=list --runner=DataflowRunner \ No newline at end of file diff --git a/sdks/python/apache_beam/testing/load_tests/build.gradle b/sdks/python/apache_beam/testing/load_tests/build.gradle index 73699e89c704..39c7ce04d0f1 100644 --- a/sdks/python/apache_beam/testing/load_tests/build.gradle +++ b/sdks/python/apache_beam/testing/load_tests/build.gradle @@ -47,6 +47,11 @@ task run(type: Exec, dependsOn: installGcpTest) { dependsOn ':sdks:python:sdist' loadTestArgs +=" --sdk_location=${files(configurations.distTarBall.files).singleFile}" } + if (!loadTestArgs.contains('--sdk_container_image') && !loadTestArgs.contains('--worker_harness_container_image')) { + loadTestArgs +=" --sdk_container_image=gcr.io/apache-beam-testing/beam-sdk/beam_python${project.ext.pythonVersion}_sdk:${project.sdk_version}" + } + } else if (loadTestArgs.contains('--environment_type=DOCKER') && !loadTestArgs.contains('--environment_config')) { + loadTestArgs +=" --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python${project.ext.pythonVersion}_sdk:${project.sdk_version}" } String requirementsTxtFileArg = project.findProperty(requirementsTxtFileProperty) ?: null diff --git a/sdks/python/apache_beam/testing/synthetic_pipeline.py b/sdks/python/apache_beam/testing/synthetic_pipeline.py index efc426ed0ff4..7070dc04ea02 100644 --- a/sdks/python/apache_beam/testing/synthetic_pipeline.py +++ b/sdks/python/apache_beam/testing/synthetic_pipeline.py @@ -478,7 +478,8 @@ def read(self, range_tracker): index = range_tracker.start_position() generator = get_generator(algorithm=self.gen_algo) while range_tracker.try_claim(index): - time.sleep(self._sleep_per_input_record_sec) + if self._sleep_per_input_record_sec > 0: + time.sleep(self._sleep_per_input_record_sec) yield self._gen_kv_pair(generator, index) index += 1 @@ -598,7 +599,8 @@ def process( cur = restriction_tracker.current_restriction().start while restriction_tracker.try_claim(cur): r = get_generator(algorithm=element.get('algorithm', None), seed=cur) - time.sleep(element['sleep_per_input_record_sec']) + if element['sleep_per_input_record_sec'] > 0: + time.sleep(element['sleep_per_input_record_sec']) yield ( r.randbytes(element['key_size']), r.randbytes(element['value_size'])) cur += 1