diff --git a/LICENSE-binary b/LICENSE-binary index 7e29e613f8057..8bbc913262c89 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -456,6 +456,7 @@ net.sf.py4j:py4j org.jpmml:pmml-model org.jpmml:pmml-schema org.threeten:threeten-extra +org.jdom:jdom2 python/lib/py4j-*-src.zip python/pyspark/cloudpickle.py @@ -504,6 +505,7 @@ Common Development and Distribution License (CDDL) 1.0 javax.activation:activation http://www.oracle.com/technetwork/java/javase/tech/index-jsp-138795.html javax.xml.stream:stax-api https://jcp.org/en/jsr/detail?id=173 javax.transaction:javax.transaction-api +javax.xml.bind:jaxb-api Common Development and Distribution License (CDDL) 1.1 diff --git a/NOTICE-binary b/NOTICE-binary index 4ce8bf2f86b2a..95653c6f49a07 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -917,6 +917,9 @@ This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin g Package (jaspell): http://jaspell.sourceforge.net/ License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) +This product includes software developed by the JDOM Project (http://www.jdom.org/) +License: https://raw.githubusercontent.com/hunterhacker/jdom/master/LICENSE.txt + The snowball stemmers in analysis/common/src/java/net/sf/snowball were developed by Martin Porter and Richard Boulton. diff --git a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Compressor.java b/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Compressor.java deleted file mode 100644 index 092ed59c6bb14..0000000000000 --- a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Compressor.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.shaded.net.jpountz.lz4; - -/** - * TODO(SPARK-36679): A temporary workaround for SPARK-36669. We should remove this after - * Hadoop 3.3.2 release which fixes the LZ4 relocation in shaded Hadoop client libraries. - * This does not need implement all net.jpountz.lz4.LZ4Compressor API, just the ones used - * by Hadoop Lz4Compressor. - */ -public final class LZ4Compressor { - - private net.jpountz.lz4.LZ4Compressor lz4Compressor; - - public LZ4Compressor(net.jpountz.lz4.LZ4Compressor lz4Compressor) { - this.lz4Compressor = lz4Compressor; - } - - public void compress(java.nio.ByteBuffer src, java.nio.ByteBuffer dest) { - lz4Compressor.compress(src, dest); - } -} diff --git a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Factory.java b/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Factory.java deleted file mode 100644 index 61829b2728bce..0000000000000 --- a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Factory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.shaded.net.jpountz.lz4; - -/** - * TODO(SPARK-36679): A temporary workaround for SPARK-36669. We should remove this after - * Hadoop 3.3.2 release which fixes the LZ4 relocation in shaded Hadoop client libraries. - * This does not need implement all net.jpountz.lz4.LZ4Factory API, just the ones used by - * Hadoop Lz4Compressor. - */ -public final class LZ4Factory { - - private net.jpountz.lz4.LZ4Factory lz4Factory; - - public LZ4Factory(net.jpountz.lz4.LZ4Factory lz4Factory) { - this.lz4Factory = lz4Factory; - } - - public static LZ4Factory fastestInstance() { - return new LZ4Factory(net.jpountz.lz4.LZ4Factory.fastestInstance()); - } - - public LZ4Compressor highCompressor() { - return new LZ4Compressor(lz4Factory.highCompressor()); - } - - public LZ4Compressor fastCompressor() { - return new LZ4Compressor(lz4Factory.fastCompressor()); - } - - public LZ4SafeDecompressor safeDecompressor() { - return new LZ4SafeDecompressor(lz4Factory.safeDecompressor()); - } -} diff --git a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4SafeDecompressor.java b/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4SafeDecompressor.java deleted file mode 100644 index cd3dd6f060f52..0000000000000 --- a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4SafeDecompressor.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.shaded.net.jpountz.lz4; - -/** - * TODO(SPARK-36679): A temporary workaround for SPARK-36669. We should remove this after - * Hadoop 3.3.2 release which fixes the LZ4 relocation in shaded Hadoop client libraries. - * This does not need implement all net.jpountz.lz4.LZ4SafeDecompressor API, just the ones - * used by Hadoop Lz4Decompressor. - */ -public final class LZ4SafeDecompressor { - private net.jpountz.lz4.LZ4SafeDecompressor lz4Decompressor; - - public LZ4SafeDecompressor(net.jpountz.lz4.LZ4SafeDecompressor lz4Decompressor) { - this.lz4Decompressor = lz4Decompressor; - } - - public void decompress(java.nio.ByteBuffer src, java.nio.ByteBuffer dest) { - lz4Decompressor.decompress(src, dest); - } -} diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 9eb89065e8718..41b1deeeca1b3 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -6,11 +6,10 @@ ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar -aliyun-java-sdk-core/3.4.0//aliyun-java-sdk-core-3.4.0.jar -aliyun-java-sdk-ecs/4.2.0//aliyun-java-sdk-ecs-4.2.0.jar -aliyun-java-sdk-ram/3.0.0//aliyun-java-sdk-ram-3.0.0.jar -aliyun-java-sdk-sts/3.0.0//aliyun-java-sdk-sts-3.0.0.jar -aliyun-sdk-oss/3.4.1//aliyun-sdk-oss-3.4.1.jar +aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar +aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar +aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar +aliyun-sdk-oss/3.13.0//aliyun-sdk-oss-3.13.0.jar annotations/17.0.0//annotations-17.0.0.jar antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr4-runtime/4.8//antlr4-runtime-4.8.jar @@ -26,7 +25,7 @@ automaton/1.11-8//automaton-1.11-8.jar avro-ipc/1.11.0//avro-ipc-1.11.0.jar avro-mapred/1.11.0//avro-mapred-1.11.0.jar avro/1.11.0//avro-1.11.0.jar -aws-java-sdk-bundle/1.11.901//aws-java-sdk-bundle-1.11.901.jar +aws-java-sdk-bundle/1.11.1026//aws-java-sdk-bundle-1.11.1026.jar azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar azure-storage/7.0.1//azure-storage-7.0.1.jar @@ -67,18 +66,18 @@ generex/1.0.2//generex-1.0.2.jar gmetric4j/1.0.10//gmetric4j-1.0.10.jar gson/2.2.4//gson-2.2.4.jar guava/14.0.1//guava-14.0.1.jar -hadoop-aliyun/3.3.1//hadoop-aliyun-3.3.1.jar -hadoop-annotations/3.3.1//hadoop-annotations-3.3.1.jar -hadoop-aws/3.3.1//hadoop-aws-3.3.1.jar -hadoop-azure-datalake/3.3.1//hadoop-azure-datalake-3.3.1.jar -hadoop-azure/3.3.1//hadoop-azure-3.3.1.jar -hadoop-client-api/3.3.1//hadoop-client-api-3.3.1.jar -hadoop-client-runtime/3.3.1//hadoop-client-runtime-3.3.1.jar -hadoop-cloud-storage/3.3.1//hadoop-cloud-storage-3.3.1.jar -hadoop-cos/3.3.1//hadoop-cos-3.3.1.jar -hadoop-openstack/3.3.1//hadoop-openstack-3.3.1.jar +hadoop-aliyun/3.3.2//hadoop-aliyun-3.3.2.jar +hadoop-annotations/3.3.2//hadoop-annotations-3.3.2.jar +hadoop-aws/3.3.2//hadoop-aws-3.3.2.jar +hadoop-azure-datalake/3.3.2//hadoop-azure-datalake-3.3.2.jar +hadoop-azure/3.3.2//hadoop-azure-3.3.2.jar +hadoop-client-api/3.3.2//hadoop-client-api-3.3.2.jar +hadoop-client-runtime/3.3.2//hadoop-client-runtime-3.3.2.jar +hadoop-cloud-storage/3.3.2//hadoop-cloud-storage-3.3.2.jar +hadoop-cos/3.3.2//hadoop-cos-3.3.2.jar +hadoop-openstack/3.3.2//hadoop-openstack-3.3.2.jar hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar -hadoop-yarn-server-web-proxy/3.3.1//hadoop-yarn-server-web-proxy-3.3.1.jar +hadoop-yarn-server-web-proxy/3.3.2//hadoop-yarn-server-web-proxy-3.3.2.jar hive-beeline/2.3.9//hive-beeline-2.3.9.jar hive-cli/2.3.9//hive-cli-2.3.9.jar hive-common/2.3.9//hive-common-2.3.9.jar @@ -97,9 +96,9 @@ hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar -htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar httpclient/4.5.13//httpclient-4.5.13.jar httpcore/4.4.14//httpcore-4.4.14.jar +ini4j/0.5.4//ini4j-0.5.4.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.5.0//ivy-2.5.0.jar jackson-annotations/2.13.1//jackson-annotations-2.13.1.jar @@ -121,10 +120,11 @@ janino/3.0.16//janino-3.0.16.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javolution/5.5.1//javolution-5.5.1.jar +jaxb-api/2.2.11//jaxb-api-2.2.11.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar jcl-over-slf4j/1.7.32//jcl-over-slf4j-1.7.32.jar jdo-api/3.0.1//jdo-api-3.0.1.jar -jdom/1.1//jdom-1.1.jar +jdom2/2.0.6//jdom2-2.0.6.jar jersey-client/2.34//jersey-client-2.34.jar jersey-common/2.34//jersey-common-2.34.jar jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar @@ -204,6 +204,9 @@ objenesis/3.2//objenesis-3.2.jar okhttp/3.12.12//okhttp-3.12.12.jar okio/1.14.0//okio-1.14.0.jar opencsv/2.3//opencsv-2.3.jar +opentracing-api/0.33.0//opentracing-api-0.33.0.jar +opentracing-noop/0.33.0//opentracing-noop-0.33.0.jar +opentracing-util/0.33.0//opentracing-util-0.33.0.jar orc-core/1.7.3//orc-core-1.7.3.jar orc-mapreduce/1.7.3//orc-mapreduce-1.7.3.jar orc-shims/1.7.3//orc-shims-1.7.3.jar diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 976ce02e9ea8d..3ba96055ae05f 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -267,6 +267,13 @@ com.google.guava guava + + + org.jacoco + org.jacoco.agent + + 3.3.2 2.5.0 ${hadoop.version} 3.6.2 @@ -3427,6 +3428,7 @@ hadoop-2 + 2.7.4 2.7.1 2.4 diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index f77bc5c284ec7..b045d4615d3c4 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -48,7 +48,12 @@ object MimaExcludes { // [SPARK-37780][SQL] QueryExecutionListener support SQLConf as constructor parameter ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.util.ExecutionListenerManager.this"), // [SPARK-37786][SQL] StreamingQueryListener support use SQLConf.get to get corresponding SessionState's SQLConf - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryManager.this") + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryManager.this"), + + // [SPARK-37600][BUILD] Upgrade to Hadoop 3.3.2 + ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Compressor"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Factory"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4SafeDecompressor") ) // Exclude rules for 3.2.x from 3.1.1 diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 671b80f4b8abe..15c172a6e75c2 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -69,7 +69,7 @@ private[hive] object IsolatedClientLoader extends Logging { // If the error message contains hadoop, it is probably because the hadoop // version cannot be resolved. val fallbackVersion = if (VersionUtils.isHadoop3) { - "3.3.1" + "3.3.2" } else { "2.7.4" }