Skip to content

Commit a877f3f

Browse files
ddebowczyk92ddebowczyk92
committed
[runners-flink] Add support for Apache Flink 2.1.3
1 parent 64bb910 commit a877f3f

11 files changed

Lines changed: 152 additions & 9 deletions

File tree

‎.github/actions/setup-default-test-properties/test-properties.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
},
1515
"JavaTestProperties": {
1616
"SUPPORTED_VERSIONS": ["8", "11", "17", "21", "25"],
17-
"FLINK_VERSIONS": ["1.17", "1.18", "1.19", "1.20", "2.0"],
17+
"FLINK_VERSIONS": ["1.17", "1.18", "1.19", "1.20", "2.0", "2.1"],
1818
"SPARK_VERSIONS": ["3"]
1919
},
2020
"GoTestProperties": {

‎.github/workflows/run_rc_validation_java_quickstart.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ jobs:
8888
- name: Run QuickStart Java Flink Runner
8989
uses: ./.github/actions/gradle-command-self-hosted-action
9090
with:
91-
gradle-command: :runners:flink:2.0:runQuickstartJavaFlinkLocal
91+
gradle-command: :runners:flink:2.1:runQuickstartJavaFlinkLocal
9292
arguments: |
9393
-Prepourl=${{ env.APACHE_REPO_URL }} \
9494
-Pver=${{ env.RELEASE_VERSION }}

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ docker_image_default_repo_root=apache
3939
docker_image_default_repo_prefix=beam_
4040

4141
# supported flink versions
42-
flink_versions=1.17,1.18,1.19,1.20,2.0
42+
flink_versions=1.17,1.18,1.19,1.20,2.0,2.1
4343
# supported spark versions
4444
spark_versions=3,4
4545
# supported python versions

‎runners/flink/2.1/build.gradle‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
project.ext {
20+
flink_major = '2.1'
21+
flink_version = '2.1.3'
22+
excluded_files = [
23+
'main': [
24+
// Used by DataSet API only
25+
"org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapter.java",
26+
"org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java",
27+
"org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java",
28+
"org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java",
29+
"org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java",
30+
// Moved to org.apache.flink.runtime.state.StateBackendFactory
31+
"org/apache/beam/runners/flink/FlinkStateBackendFactory.java",
32+
],
33+
'test': [
34+
// Used by DataSet API only
35+
"org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapterTest.java",
36+
"org/apache/beam/runners/flink/batch/NonMergingGroupByKeyTest.java",
37+
"org/apache/beam/runners/flink/batch/ReshuffleTest.java",
38+
]
39+
]
40+
}
41+
42+
// Load the main build script which contains all build logic.
43+
apply from: "../flink_runner.gradle"
44+
45+
// Flink 2.1 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
46+
// Resolve capability conflict by preferring Flink's version
47+
configurations.all {
48+
resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java') {
49+
selectHighestVersion()
50+
}
51+
}

‎runners/flink/2.1/job-server-container/build.gradle‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
def basePath = '../../job-server-container'
20+
21+
project.ext {
22+
resource_path = basePath
23+
}
24+
25+
// Load the main build script which contains all build logic.
26+
apply from: "$basePath/flink_job_server_container.gradle"

‎runners/flink/2.1/job-server/build.gradle‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
def basePath = '../../job-server'
20+
21+
project.ext {
22+
// Look for the source code in the parent module
23+
main_source_dirs = ["$basePath/src/main/java"]
24+
test_source_dirs = ["$basePath/src/test/java"]
25+
main_resources_dirs = ["$basePath/src/main/resources"]
26+
test_resources_dirs = ["$basePath/src/test/resources"]
27+
archives_base_name = 'beam-runners-flink-2.1-job-server'
28+
}
29+
30+
// Load the main build script which contains all build logic.
31+
apply from: "$basePath/flink_job_server.gradle"
32+
33+
// Flink 2.1 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
34+
// Resolve capability conflict by preferring Flink's version
35+
configurations.all {
36+
resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java') {
37+
selectHighestVersion()
38+
}
39+
}

‎runners/flink/2.1/src/test/resources/flink-test-config.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
19+
taskmanager:
20+
memory:
21+
network:
22+
max: 2gb
23+
fraction: '0.2'
24+
managed:
25+
size: 1gb
26+
parallelism:
27+
default: '23'

‎sdks/go/examples/wasm/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ cd $BEAM_HOME
6868
Expected output should include the following, from which you acquire the latest flink runner version.
6969

7070
```shell
71-
'flink_versions: 1.17,1.18,1.19,1.20'
71+
'flink_versions: 1.17,1.18,1.19,1.20,2.0,2.1'
7272
```
7373

74-
#### 2. Set to the latest flink runner version i.e. 1.16
74+
#### 2. Set to the latest flink runner version i.e. 2.1
7575

7676
```shell
77-
FLINK_VERSION=1.16
77+
FLINK_VERSION=2.1
7878
```
7979

8080
#### 3. In a separate terminal, start the flink runner (It should take a few minutes on the first execution)

‎sdks/go/test/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ task flinkValidatesRunner {
8989
doFirst {
9090
// Copy Flink conf file
9191
copy {
92-
from "${project.rootDir}/runners/flink/2.0/src/test/resources/flink-test-config.yaml"
92+
from "${project.rootDir}/runners/flink/${flinkVersion}/src/test/resources/flink-test-config.yaml"
9393
into "${project.buildDir}/flink-conf"
9494

9595
// Rename the file during the copy process

‎sdks/python/apache_beam/options/pipeline_options.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2030,7 +2030,7 @@ def _add_argparse_args(cls, parser):
20302030
class FlinkRunnerOptions(PipelineOptions):
20312031

20322032
# These should stay in sync with gradle.properties.
2033-
PUBLISHED_FLINK_VERSIONS = ['1.17', '1.18', '1.19', '1.20', '2.0']
2033+
PUBLISHED_FLINK_VERSIONS = ['1.17', '1.18', '1.19', '1.20', '2.0', '2.1']
20342034

20352035
@classmethod
20362036
def _add_argparse_args(cls, parser):

0 commit comments

Comments
 (0)