diff --git a/assembly/pom.xml b/assembly/pom.xml index 6263ee4932d51..772e0f1c2f490 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index bd2f88b0ee8ed..3cc3bae48d528 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 85a4631729dc1..413deda400692 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index bca0537e4d7b4..fbe0bb1d64071 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index a0b7e88e6c935..c5a90affce5a7 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 613561b92a954..25119c85db780 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index c749387db4975..eb0c99e9416ea 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index e0d48291afa2f..c4c13ec4400c3 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index dd4fa242f3064..5e8c54b72f3f3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 99735631d7b78..127e59d0aed0a 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -85,7 +85,7 @@ GIT_REF=${GIT_REF:-master} RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/spark" GPG="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback" -NEXUS_ROOT=https://repository.apache.org/service/local/staging +NEXUS_ROOT=https://jfrog.fkinternal.com/artifactory/maven_virtual/ NEXUS_PROFILE=d63f592e7eac0 # Profile for Spark staging uploads BASE_DIR=$(pwd) diff --git a/dev/create-release/vote.tmpl b/dev/create-release/vote.tmpl index aa22a9c516535..1072c288ed328 100644 --- a/dev/create-release/vote.tmpl +++ b/dev/create-release/vote.tmpl @@ -18,7 +18,7 @@ Signatures used for Spark RCs can be found in this file: https://dist.apache.org/repos/dist/dev/spark/KEYS The staging repository for this release can be found at: -https://repository.apache.org/content/repositories/orgapachespark-{repo_id}/ +https://jfrog.fkinternal.com/artifactory/maven_virtual/ The documentation corresponding to this release can be found at: https://dist.apache.org/repos/dist/dev/spark/{tag}-docs/ diff --git a/examples/pom.xml b/examples/pom.xml index c9bcfe5d9e753..768fc045150c3 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index c678e0c689a05..8639af95a7505 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 0a3e6837421bd..afac1b9120460 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index 0376d0672dd92..764c8b01b9a72 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 79778fb4ea066..fd9abddefc878 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml index bdae71dd9a5d3..5e4fe347020c7 100644 --- a/external/kafka-0-10-token-provider/pom.xml +++ b/external/kafka-0-10-token-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 77aa1b5afba1f..f4239457385b8 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 655ec2680fcb5..46492edcfc36d 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 7d27794b982da..bef1960aa89e5 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index facef36532c25..b9bb17987efe8 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index ed81876459a84..6a8f0dc2ee6df 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index d66aae994f71d..98abfdd50b366 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 2097cbd9890cb..45416ffaac684 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index efd5ddaa1db05..1e6e3f5665147 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 5ee35f537d5f5..52ac1e7f7e34f 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 3f114fd1a3f8d..fc8fb37b05673 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 pom Spark Project Parent POM http://spark.apache.org/ @@ -279,6 +279,14 @@ + flipkart-libs-snapshots-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + + + flipkart-libs-release-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + + central Maven Repository https://repo.maven.apache.org/maven2 @@ -306,6 +314,14 @@ false + + flipkart-libs-snapshots-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + + + flipkart-libs-release-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + central https://repo.maven.apache.org/maven2 @@ -3316,8 +3332,8 @@ snapshots-and-staging - https://repository.apache.org/content/groups/staging/ - https://repository.apache.org/content/repositories/snapshots/ + https://jfrog.fkinternal.com/artifactory/maven_virtual + https://jfrog.fkinternal.com/artifactory/maven_virtual/ @@ -3383,4 +3399,16 @@ + + + fk-jfrog-snapshot + libs-snapshot + https://jfrog.fkinternal.com/artifactory/maven_virtual + + + fk-jfrog-release + libs-rel + https://jfrog.fkinternal.com/artifactory/maven_virtual + + diff --git a/repl/pom.xml b/repl/pom.xml index fbe6a84ad922f..8731ca01dbc5e 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 5e997b217644a..f0c43c6d5ce4c 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 9e8963d66d964..4072941fe8e65 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 946416e32cf56..e32db53cfa6ca 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index b2c635ca78543..4fa081d09ebe6 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 618faef2d58b3..af2d4af929447 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -54,6 +54,7 @@ import org.apache.spark.deploy.{SparkApplication, SparkHadoopUtil} import org.apache.spark.deploy.security.HadoopDelegationTokenManager import org.apache.spark.deploy.yarn.ResourceRequestHelper._ import org.apache.spark.deploy.yarn.config._ +import org.apache.spark.deploy.yarn.propertymodifier.QueueEnforcer import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ import org.apache.spark.internal.config.Python._ @@ -241,6 +242,39 @@ private[spark] class Client( cleanupStagingDirInternal() } + private def getInitiator: String = { + var user: String = null + try { + user = JobInitiatorFetcher.getJobInitiator + } catch { + case e: Throwable => + throw new RuntimeException("User couldn't be set. ", e) + } + user + } + + @throws[IOException] + private def getEnforcedQueue(queue: String): String = { + log.info("Input queue name: " + queue) + val queueEnforcerClassName: String = sparkConf.get(QUEUE_ENFORCER_CLASS) + + if (queueEnforcerClassName == null) { + throw new RuntimeException("Queue enforcer class is null.") + } + + var queueEnforcer: QueueEnforcer = null + try {queueEnforcer = Utils.classForName(queueEnforcerClassName) + .newInstance.asInstanceOf[QueueEnforcer] + } catch { + case e: Throwable => + throw new RuntimeException("Couldn't enforce queue. ", e) + } + log.info("queue enforcer class: " + queueEnforcerClassName) + val enforcedQueue: String = queueEnforcer.getEnforcedQueue(queue, getInitiator, sparkConf) + log.info("Enforced queue name: " + enforcedQueue) + enforcedQueue + } + /** * Set up the context for submitting our ApplicationMaster. * This uses the YarnClientApplication not available in the Yarn alpha API. @@ -260,7 +294,8 @@ private[spark] class Client( logDebug(s"AM resources: $amResources") val appContext = newApp.getApplicationSubmissionContext appContext.setApplicationName(sparkConf.get("spark.app.name", "Spark")) - appContext.setQueue(sparkConf.get(QUEUE_NAME)) + log.info("setting queue") + appContext.setQueue(getEnforcedQueue(sparkConf.get(QUEUE_NAME))) appContext.setAMContainerSpec(containerContext) appContext.setApplicationType(sparkConf.get(APPLICATION_TYPE)) @@ -1267,6 +1302,21 @@ private[spark] class Client( } +private object JobInitiatorFetcher { + val SUDO_USER_PROP_KEY = "SUDO_USER" + val USER = "USER" + + @throws[IOException] + @throws[InterruptedException] + def getJobInitiator: String = { + val sudoUser = Option(System.getenv.get(SUDO_USER_PROP_KEY)) + val user = Option(System.getenv.get(USER)) + if (sudoUser.isDefined) return sudoUser.get + else if (user.isDefined) return user.get + null + } +} + private object Client extends Logging { // Alias for the user jar diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala index 89a4af2d2a741..14baa9df0b70d 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala @@ -101,6 +101,11 @@ package object config extends Logging { .stringConf .createWithDefault("default") + private[spark] val QUEUE_ENFORCER_CLASS = ConfigBuilder("spark.queue.enforcer.class") + .doc("Fully Qualified class name for concrete impl to be used to enforce queue") + .stringConf + .createWithDefault(null) + private[spark] val HISTORY_SERVER_ADDRESS = ConfigBuilder("spark.yarn.historyServer.address") .version("1.0.0") .stringConf diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/QueueEnforcer.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/QueueEnforcer.scala new file mode 100644 index 0000000000000..53c2bc156d584 --- /dev/null +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/QueueEnforcer.scala @@ -0,0 +1,24 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.spark.deploy.yarn.propertymodifier + +import org.apache.spark.SparkConf + +trait QueueEnforcer { + def getEnforcedQueue(existingQueue: String, initiator: String, conf: SparkConf): String +} \ No newline at end of file diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/ReturnExistingQueue.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/ReturnExistingQueue.scala new file mode 100644 index 0000000000000..a32be552a112a --- /dev/null +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/ReturnExistingQueue.scala @@ -0,0 +1,25 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.spark.deploy.yarn.propertymodifier + +import org.apache.spark.SparkConf + +class ReturnExistingQueue extends QueueEnforcer { + override def getEnforcedQueue(existingQueue: String, initiator: String, conf: SparkConf): + String = existingQueue +} \ No newline at end of file diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 08bb504da9c87..eb735635fffe1 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index a4043d9cb3a7b..fe53f13c1dea4 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala index 2671682e18f31..678edca7d30ad 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ import org.apache.spark.util.{SerializableConfiguration, Utils} +import org.apache.spark.sql.catalyst.util.quoteIdentifier private[sql] object OrcFileFormat { private def checkFieldName(name: String): Unit = { @@ -161,6 +162,10 @@ class OrcFileFormat OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(hadoopConf, sqlConf.caseSensitiveAnalysis) + val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema) + OrcConf.MAPRED_INPUT_SCHEMA.setString(hadoopConf, resultSchemaString) + OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(hadoopConf, sqlConf.caseSensitiveAnalysis) + val broadcastedConf = sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) val isCaseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis @@ -193,10 +198,10 @@ class OrcFileFormat } val (requestedColIds, canPruneCols) = resultedColPruneInfo.get - val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols, - dataSchema, resultSchema, partitionSchema, conf) - assert(requestedColIds.length == requiredSchema.length, - "[BUG] requested column IDs do not match required schema") + // val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols, + // dataSchema, resultSchema, partitionSchema, conf) + // assert(requestedColIds.length == requiredSchema.length, + // "[BUG] requested column IDs do not match required schema") val taskConf = new Configuration(conf) val fileSplit = new FileSplit(filePath, file.start, file.length, Array.empty) @@ -258,4 +263,22 @@ class OrcFileFormat case _ => false } + + /** + * Given a `StructType` object, this methods converts it to corresponding string representation + * in ORC. + */ + def orcTypeDescriptionString(dt: DataType): String = dt match { + case s: StructType => + val fieldTypes = s.fields.map { f => + s"${quoteIdentifier(f.name)}:${orcTypeDescriptionString(f.dataType)}" + } + s"struct<${fieldTypes.mkString(",")}>" + case a: ArrayType => + s"array<${orcTypeDescriptionString(a.elementType)}>" + case m: MapType => + s"map<${orcTypeDescriptionString(m.keyType)},${orcTypeDescriptionString(m.valueType)}>" + case _ => dt.catalogString + } + } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala index 623f4f7a54d00..c34e48945aebb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala @@ -183,7 +183,6 @@ object OrcUtils extends Logging { if (matchedOrcFields.size > 1) { // Need to fail if there is ambiguity, i.e. more than one field is matched. val matchedOrcFieldsString = matchedOrcFields.mkString("[", ", ", "]") - reader.close() throw new RuntimeException(s"""Found duplicate field(s) "$requiredFieldName": """ + s"$matchedOrcFieldsString in case-insensitive mode") } else { @@ -203,6 +202,7 @@ object OrcUtils extends Logging { writer.addUserMetadata(SPARK_VERSION_METADATA_KEY, UTF_8.encode(SPARK_VERSION_SHORT)) } + /** * Given a `StructType` object, this methods converts it to corresponding string representation * in ORC. @@ -220,6 +220,7 @@ object OrcUtils extends Logging { case _ => dt.catalogString } + /** * Returns the result schema to read from ORC file. In addition, It sets * the schema string to 'orc.mapred.input.schema' so ORC reader can use later. @@ -246,4 +247,5 @@ object OrcUtils extends Logging { OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString) resultSchemaString } + } diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index addb2dca0c463..cd0ce129090ae 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index b07e34f1cf86e..8eaae41005993 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala index fcf7febe33121..a19ce51223ad8 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala @@ -150,6 +150,52 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton { } } + test("FDG-4295 - ORC hive table column reordering - Lookup by position") { + withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> "native") { + withTempPath { dir => + withTable("test_user_orc") { + // Case - Read as is + sql( + s"""create external table test_user_orc (`id` long, + |`user` struct>) + |stored as orc location "${dir.toURI}"""".stripMargin) + sql( + s"""insert into test_user_orc select 1, NAMED_STRUCT('age', 10, 'name', + |NAMED_STRUCT('first', 'myfirst', 'last', 'mylast'))""".stripMargin) + val query = "select `id`,`user`.age,`user`.name.first,`user`.name.last from test_user_orc" + val df0 = spark.sql(query) + sql("drop table test_user_orc") + + // Case:0 - depth-0 column re-order + sql( + s"""create external table test_user_orc ( + |`user` struct>, + |`id` long) stored as orc location "${dir.toURI}"""".stripMargin) + val df1 = spark.sql(query) + checkAnswer(df1, df0) + sql("drop table test_user_orc") + + // Case:1 - sub-struct column re-order + sql( + s"""create external table test_user_orc (`id` long, + |`user` struct, age: int>) + |stored as orc location "${dir.toURI}"""".stripMargin) + val df2 = spark.sql(query) + checkAnswer(df2, df0) + sql("drop table test_user_orc") + + // Case:2 - all column re-order + sql( + s"""create external table test_user_orc ( + |`user` struct, age: int>, + |`id` long) stored as orc location "${dir.toURI}"""".stripMargin) + val df3 = spark.sql(query) + checkAnswer(df3, df0) + } + } + } + } + test("SPARK-20728 Make ORCFileFormat configurable between sql/hive and sql/core") { Seq( ("native", classOf[org.apache.spark.sql.execution.datasources.orc.OrcFileFormat]), diff --git a/streaming/pom.xml b/streaming/pom.xml index e64e269cb39d7..d53c33c955788 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 3b2314dc8d370..56f50ec258f92 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml