diff --git a/testing/hive3.1-hive/Dockerfile b/testing/hive3.1-hive/Dockerfile index c274e498..6e462f07 100644 --- a/testing/hive3.1-hive/Dockerfile +++ b/testing/hive3.1-hive/Dockerfile @@ -42,6 +42,8 @@ ARG HADOOP_BINARY_SHA256=1c02ccc60a09c63a48dc4234ffd3aed1b75e5a1f2b49d60927eda11 ARG HIVE_VERSION=3.1.3 ARG HIVE_BINARY_SHA256=0c9b6a6359a7341b6029cc9347435ee7b379f93846f779d710b13f795b54bb16 +ARG GCS_CONNECTOR_VERSION=hadoop2-2.2.24 + ARG HADOOP_BINARY_PATH=https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz ARG HIVE_BINARY_PATH=https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz @@ -60,6 +62,9 @@ RUN mkdir /opt/hive/auxlib && \ curl -fLsS -o /opt/hive/auxlib/aws-java-sdk-core-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/$AWS_SDK_VERSION/aws-java-sdk-core-$AWS_SDK_VERSION.jar && \ curl -fLsS -o /opt/hive/auxlib/aws-java-sdk-s3-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/$AWS_SDK_VERSION/aws-java-sdk-s3-$AWS_SDK_VERSION.jar +RUN curl -fLsS -o /opt/hadoop/share/hadoop/common/lib/gcs-connector-$GCS_CONNECTOR_VERSION-shaded.jar \ + https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/$GCS_CONNECTOR_VERSION/gcs-connector-$GCS_CONNECTOR_VERSION-shaded.jar + ENV HADOOP_HOME=/opt/hadoop ENV HIVE_HOME=/opt/hive ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/* diff --git a/testing/spark4-iceberg/Dockerfile b/testing/spark4-iceberg/Dockerfile index 5e40c86e..70d186b6 100644 --- a/testing/spark4-iceberg/Dockerfile +++ b/testing/spark4-iceberg/Dockerfile @@ -18,6 +18,7 @@ ARG HADOOP_VERSION=3 ARG ICEBERG_VERSION=1.11.0 # ICEBERG_JAR_VERSION is derived from: _ ARG ICEBERG_JAR_VERSION=4.0_2.13 +ARG GCS_CONNECTOR_VERSION=hadoop2-2.2.24 ARG POSTGRESQL_JAR_VERSION=42.3.5 ARG SPARK_ARTIFACT="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" @@ -44,7 +45,7 @@ RUN wget -nv https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/8. RUN wget -nv https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar RUN wget -nv https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar # install Google Hadoop connector so we can access gcs -RUN wget -nv https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-latest.jar +RUN wget -nv "https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/${GCS_CONNECTOR_VERSION}/gcs-connector-${GCS_CONNECTOR_VERSION}-shaded.jar" # install Iceberg RUN wget -nv "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_JAR_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_JAR_VERSION}-${ICEBERG_VERSION}.jar"