From 31011f07d07b173e91c9c3af8e020876b4a6a326 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 1 Oct 2024 00:36:13 +0800 Subject: [PATCH 1/2] [TOREE-557] Bump Spark 3.5 --- Dockerfile | 4 ++-- Dockerfile.toree-dev | 2 +- Makefile | 2 +- build.sbt | 4 ++-- etc/kernel.json | 2 +- project/Dependencies.scala | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index bec2dd02..49d83ce9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \ npm install -g bower # for Apache Spark demos -ENV APACHE_SPARK_VERSION 3.4.4 +ENV APACHE_SPARK_VERSION 3.5.8 ARG SCALA_VERSION=2.12 RUN apt-get -y update && \ @@ -62,7 +62,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ENV SPARK_HOME /usr/local/spark -ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.5-src.zip +ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip ENV PYSPARK_PYTHON /home/main/anaconda2/envs/python3/bin/python ENV R_LIBS_USER $SPARK_HOME/R/lib diff --git a/Dockerfile.toree-dev b/Dockerfile.toree-dev index 8cb291f9..68e983f0 100644 --- a/Dockerfile.toree-dev +++ b/Dockerfile.toree-dev @@ -23,7 +23,7 @@ FROM jupyter/all-spark-notebook USER root # Spark dependencies -ARG APACHE_SPARK_VERSION=3.4.4 +ARG APACHE_SPARK_VERSION=3.5.8 ARG SCALA_VERSION=2.12 RUN JAVA_8=`update-alternatives --list java | grep java-1.8.0-openjdk` || echo $JAVA_8 && \ diff --git a/Makefile b/Makefile index 2a8f4525..918714d6 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ IS_SNAPSHOT?=true SNAPSHOT:=-SNAPSHOT endif -APACHE_SPARK_VERSION?=3.4.4 +APACHE_SPARK_VERSION?=3.5.8 SCALA_VERSION?=2.12 IMAGE?=jupyter/all-spark-notebook:latest EXAMPLE_IMAGE?=apache/toree-examples diff --git a/build.sbt b/build.sbt index 626dcbbe..5061931d 100644 --- a/build.sbt +++ b/build.sbt @@ -18,7 +18,7 @@ import scala.util.Properties import sbtassembly.AssemblyOption -lazy val scala212 = "2.12.17" +lazy val scala212 = "2.12.18" lazy val scala213 = "2.13.8" lazy val defaultScalaVersion = sys.env.get("SCALA_VERSION") match { case Some("2.13") => scala213 @@ -34,7 +34,7 @@ ThisBuild / crossScalaVersions := Seq(scala212, scala213) ThisBuild / scalaVersion := defaultScalaVersion ThisBuild / Dependencies.sparkVersion := { val envVar = "APACHE_SPARK_VERSION" - val defaultVersion = "3.4.4" + val defaultVersion = "3.5.8" Properties.envOrNone(envVar) match { case None => diff --git a/etc/kernel.json b/etc/kernel.json index 868215ac..98808659 100644 --- a/etc/kernel.json +++ b/etc/kernel.json @@ -4,7 +4,7 @@ }, "display_name": "Apache Toree (development)", "env": { - "PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9.5-src.zip", + "PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip", "SPARK_HOME": "/usr/local/spark", "CAPTURE_STANDARD_ERR": "true", "MAX_INTERPRETER_THREADS": "16", diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 69dc67d7..368ddcb5 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -32,14 +32,14 @@ object Dependencies { val config = "com.typesafe" % "config" % "1.4.3" // Apache v2 - val coursierVersion = "2.0.0" + val coursierVersion = "2.0.16" val coursier = "io.get-coursier" %% "coursier" % coursierVersion // Apache v2 val coursierCache = "io.get-coursier" %% "coursier-cache" % coursierVersion // Apache v2 val ivy = "org.apache.ivy" % "ivy" % "2.5.1" // Apache v2 - // use the same jackson version in test than the one provided at runtime by Spark 3.4.x - val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.14.2" // Apache v2 + // use the same jackson version in test than the one provided at runtime by Spark 3.5.x + val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.2" // Apache v2 val jeroMq = "org.zeromq" % "jeromq" % "0.5.3" // MPL v2 @@ -57,7 +57,7 @@ object Dependencies { val scalaTestMockito = "org.scalatestplus" %% "mockito-4-11" % "3.2.16.0" // Apache v2 val mockitoInline = "org.mockito" % "mockito-inline" % "4.11.0" // MIT - val slf4jApi = "org.slf4j" % "slf4j-api" % "2.0.6" // MIT + val slf4jApi = "org.slf4j" % "slf4j-api" % "2.0.7" // MIT val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value } // Apache v2 From ff78567f7f3cad071149961832aa3091ac3822f9 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 7 Oct 2025 11:29:03 +0800 Subject: [PATCH 2/2] asm9 ^ Conflicts: ^ project/Dependencies.scala --- plugins/build.sbt | 3 +++ .../scala/org/apache/toree/plugins/PluginSearcher.scala | 7 ++++--- project/Dependencies.scala | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/build.sbt b/plugins/build.sbt index d61c0a35..ad46733f 100644 --- a/plugins/build.sbt +++ b/plugins/build.sbt @@ -20,6 +20,9 @@ Test / fork := true // Needed for type inspection libraryDependencies ++= Seq( Dependencies.scalaReflect.value, + Dependencies.asm, + Dependencies.asmCommons, + Dependencies.asmUtil, Dependencies.clapper, Dependencies.slf4jApi, ) diff --git a/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala b/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala index 75fa6f92..5e2a661e 100644 --- a/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala +++ b/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala @@ -17,7 +17,8 @@ package org.apache.toree.plugins import java.io.File -import org.clapper.classutil.{ClassInfo, ClassFinder} +import org.clapper.classutil.{ClassFinder, ClassInfo} +import org.objectweb.asm.Opcodes import org.slf4j.LoggerFactory import scala.annotation.tailrec @@ -52,7 +53,7 @@ class PluginSearcher { * * @return The new class finder */ - protected def newClassFinder(): ClassFinder = ClassFinder(classpath) + protected def newClassFinder(): ClassFinder = ClassFinder(classpath, Some(Opcodes.ASM9)) /** * Creates a new class finder for the given paths. @@ -61,7 +62,7 @@ class PluginSearcher { * * @return The new class finder */ - protected def newClassFinder(paths: Seq[File]): ClassFinder = ClassFinder(paths) + protected def newClassFinder(paths: Seq[File]): ClassFinder = ClassFinder(paths, Some(Opcodes.ASM9)) /** * Loads all class information using the provided class finder. diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 368ddcb5..a192a309 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -26,6 +26,10 @@ object Dependencies { val pekkoSlf4j = "org.apache.pekko" %% "pekko-slf4j" % pekkoVersion // Apache v2 val pekkoTestkit = "org.apache.pekko" %% "pekko-testkit" % pekkoVersion // Apache v2 + val asmVersion = "9.9" + val asm = "org.ow2.asm" % "asm" % asmVersion // Apache v2 + val asmCommons = "org.ow2.asm" % "asm-commons" % asmVersion // Apache v2 + val asmUtil = "org.ow2.asm" % "asm-util" % asmVersion // Apache v2 val clapper = "org.clapper" %% "classutil" % "1.5.1" // Apache v2, used for detecting plugins val commonsExec = "org.apache.commons" % "commons-exec" % "1.3" // Apache v2