From f1f04b097d2ff8850426640c77dd43a53f449ad6 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 17 Oct 2024 07:27:40 -0700 Subject: [PATCH] [SPARK-49988][BUILD] Remove unused Hadoop dependency management ### What changes were proposed in this pull request? Remove unused vanilla hadoop dependency(and transitive deps) management, i.e. `hadoop-client`, `xerces:xercesImpl`, and inline deps defined in `hadoop3` because it's the only supported hadoop profile. ### Why are the changes needed? Simplify pom.xml. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass CI and verified runtime jars are not affected by running`dev/test-dependencies.sh`. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #48491 from pan3793/SPARK-49988. Authored-by: Cheng Pan Signed-off-by: Dongjoon Hyun --- LICENSE-binary | 1 - NOTICE-binary | 21 -- pom.xml | 297 ------------------ project/SparkBuild.scala | 1 - .../kubernetes/integration-tests/pom.xml | 20 +- 5 files changed, 6 insertions(+), 334 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 89826482d363a..40d28fbe71e6b 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -402,7 +402,6 @@ org.xerial.snappy:snappy-java org.yaml:snakeyaml oro:oro stax:stax-api -xerces:xercesImpl core/src/main/java/org/apache/spark/util/collection/TimSort.java core/src/main/resources/org/apache/spark/ui/static/bootstrap* diff --git a/NOTICE-binary b/NOTICE-binary index c4cfe0e9f8b31..3f36596b9d6d6 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -448,27 +448,6 @@ which has the following notices: * Alec Wysoker * Performance and memory usage improvement -The binary distribution of this product bundles binaries of -Xerces2 Java Parser 2.9.1, -which has the following notices: - * ========================================================================= - == NOTICE file corresponding to section 4(d) of the Apache License, == - == Version 2.0, in this case for the Apache Xerces Java distribution. == - ========================================================================= - - Apache Xerces Java - Copyright 1999-2007 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - Apache Commons Collections Copyright 2001-2015 The Apache Software Foundation diff --git a/pom.xml b/pom.xml index ff15f200e2bb1..2e169df7201c2 100644 --- a/pom.xml +++ b/pom.xml @@ -1418,92 +1418,6 @@ test - - org.apache.hadoop - hadoop-client - ${hadoop.version} - ${hadoop.deps.scope} - - - org.fusesource.leveldbjni - leveldbjni-all - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - io.netty - netty - - - - commons-beanutils - commons-beanutils-core - - - commons-logging - commons-logging - - - org.mockito - mockito-all - - - org.mortbay.jetty - servlet-api-2.5 - - - javax.servlet - servlet-api - - - junit - junit - - - com.sun.jersey - * - - - com.sun.jersey.jersey-test-framework - * - - - com.sun.jersey.contribs - * - - - net.java.dev.jets3t - jets3t - - - - javax.ws.rs - jsr311-api - - - org.eclipse.jetty - jetty-webapp - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.hadoop hadoop-minikdc @@ -1544,16 +1458,6 @@ ${bouncycastle.version} test - - - - xerces - xercesImpl - 2.12.2 - org.apache.avro avro @@ -1636,207 +1540,6 @@ 1.1.1 ${hadoop.deps.scope} - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - ${hadoop.deps.scope} - - - javax.servlet - servlet-api - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - commons-logging - commons-logging - - - com.sun.jersey - * - - - com.sun.jersey.jersey-test-framework - * - - - com.sun.jersey.contribs - * - - - jdk.tools - jdk.tools - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - ${hadoop.deps.scope} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - com.sun.jersey - * - - - com.sun.jersey.jersey-test-framework - * - - - com.sun.jersey.contribs - * - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.hadoop - hadoop-yarn-server-tests - ${yarn.version} - tests - test - - - org.fusesource.leveldbjni - leveldbjni-all - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - com.sun.jersey - * - - - com.sun.jersey.jersey-test-framework - * - - - com.sun.jersey.contribs - * - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - - - - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - ${yarn.version} - test - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - ${hadoop.deps.scope} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - com.sun.jersey - * - - - com.sun.jersey.jersey-test-framework - * - - - com.sun.jersey.contribs - * - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.zookeeper zookeeper diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index a87e0af0b542f..e7f7d68e98483 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1060,7 +1060,6 @@ object DependencyOverrides { lazy val guavaVersion = sys.props.get("guava.version").getOrElse("33.1.0-jre") lazy val settings = Seq( dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, - dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2", dependencyOverrides += "jline" % "jline" % "2.14.6", dependencyOverrides += "org.apache.avro" % "avro" % "1.11.3") } diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 45ce25b8e037a..cebef07821f39 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -79,6 +79,12 @@ ${project.version} test + + software.amazon.awssdk + bundle + ${aws.java.sdk.v2.version} + test + @@ -189,20 +195,6 @@ - - hadoop-3 - - true - - - - software.amazon.awssdk - bundle - ${aws.java.sdk.v2.version} - test - - - volcano