Remove Spark2 references and update dependencies

cdapio · Jan 17, 2023 · e796429 · e796429
1 parent 7f99be6
commit e796429
Show file tree

Hide file tree

Showing 50 changed files with 111 additions and 2,577 deletions.
diff --git a/cdap-api-spark2_2.11/pom.xml b/cdap-api-spark2_2.11/pom.xml
diff --git a/cdap-api-spark3_2.12/pom.xml b/cdap-api-spark3_2.12/pom.xml
@@ -47,7 +47,6 @@
     <dependency>
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
-      <version>${scala2.12.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>

diff --git a/cdap-app-fabric-tests/pom.xml b/cdap-app-fabric-tests/pom.xml
@@ -37,7 +37,7 @@ the License.
     </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
-      <artifactId>cdap-api-spark2_2.11</artifactId>
+      <artifactId>cdap-api-spark3_2.12</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
@@ -62,7 +62,7 @@ the License.
     </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
-      <artifactId>cdap-spark-core2_2.11</artifactId>
+      <artifactId>cdap-spark-core3_2.12</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>

diff --git a/.../io/cdap/cdap/internal/app/runtime/distributed/DistributedProgramRunnerTxTimeoutTest.java b/.../io/cdap/cdap/internal/app/runtime/distributed/DistributedProgramRunnerTxTimeoutTest.java
@@ -86,7 +86,7 @@ public static void setup() {
                                                       injector);
     mapreduceRunner = new DistributedMapReduceProgramRunner(cConf, yConf, null, ClusterMode.ON_PREMISE, null,
                                                             injector);
-    sparkRunner = new DistributedSparkProgramRunner(SparkCompat.SPARK2_2_11, cConf, yConf, null, null,
+    sparkRunner = new DistributedSparkProgramRunner(SparkCompat.SPARK3_2_12, cConf, yConf, null, null,
                                                     ClusterMode.ON_PREMISE, null, injector);
     workflowRunner = new DistributedWorkflowProgramRunner(cConf, yConf, null, ClusterMode.ON_PREMISE, null, null,
                                                           injector);

diff --git a/cdap-app-fabric/pom.xml b/cdap-app-fabric/pom.xml
@@ -82,6 +82,12 @@
       <groupId>io.cdap.cdap</groupId>
       <artifactId>cdap-security</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.eclipse.jetty.orbit</groupId>
+          <artifactId>javax.servlet</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
@@ -254,7 +260,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-core_2.11</artifactId>
+      <artifactId>spark-core_2.12</artifactId>
       <scope>provided</scope>
     </dependency>
   </dependencies>

diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/spark/SparkCompatReader.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/spark/SparkCompatReader.java
@@ -37,7 +37,7 @@ private SparkCompatReader() {
 
   /**
    * Read {@link SparkCompat} from the system properties, environment, or the {@link CConfiguration}.
-   * Tries to detect or falls back to {@link SparkCompat#SPARK2_2_11} if it is not defined in any place.
+   * Tries to detect or falls back to {@link SparkCompat#SPARK3_2_12} if it is not defined in any place.
    *
    * @param cConf the {@link CConfiguration} for CDAP
    * @return the configured {@link SparkCompat}
@@ -51,7 +51,7 @@ public static SparkCompat get(CConfiguration cConf) {
     compatStr = compatStr == null ? cConf.get(Constants.AppFabric.SPARK_COMPAT) : compatStr;
 
     if (compatStr == null) {
-      return SparkCompat.SPARK2_2_11;
+      return SparkCompat.SPARK3_2_12;
     }
 
     for (SparkCompat sparkCompat : SparkCompat.values()) {

diff --git a/...abric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java b/...abric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java
@@ -16,6 +16,7 @@
 
 package io.cdap.cdap.internal.capability;
 
+import com.google.common.base.Throwables;
 import com.google.common.io.Files;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
@@ -49,7 +50,6 @@
 import io.cdap.cdap.proto.id.NamespaceId;
 import io.cdap.cdap.proto.id.ProgramId;
 import io.cdap.cdap.spi.data.transaction.TransactionRunner;
-import jersey.repackaged.com.google.common.base.Throwables;
 import org.apache.twill.filesystem.Location;
 import org.apache.twill.filesystem.LocationFactory;
 import org.junit.After;