Merge remote-tracking branch 'spark/master' into shuffle-spec-direct-partition

Author: shujingyang-db

.github/workflows/maven_test.yml

@@ -205,11 +205,11 @@ jobs:
           
           if [[ "$INCLUDED_TAGS" != "" ]]; then
             ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.include.tags="$INCLUDED_TAGS" test -fae
-          elif [[ "$MODULES_TO_TEST" == "connect" ]]; then
-            ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jdbc,sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae
           elif [[ "$MODULES_TO_TEST" == "connect" && "$INPUT_BRANCH" == "branch-4.0" ]]; then
             # SPARK-53914: Remove sql/connect/client/jdbc from `-pl` for branch-4.0, this branch can be deleted after the EOL of branch-4.0.
             ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae
+          elif [[ "$MODULES_TO_TEST" == "connect" ]]; then
+            ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jdbc,sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae
           elif [[ "$EXCLUDED_TAGS" != "" ]]; then
             ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.exclude.tags="$EXCLUDED_TAGS" test -fae
           elif [[ "$MODULES_TO_TEST" == *"sql#hive-thriftserver"* ]]; then

common/unsafe/src/main/java/org/apache/spark/unsafe/types/GeographyVal.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.unsafe.types;
+
+import java.io.Serializable;
+
+// This class represents the physical type for the GEOGRAPHY data type.
+public final class GeographyVal implements Comparable<GeographyVal>, Serializable {
+
+  // The GEOGRAPHY type is implemented as a byte array. We provide `getBytes` and `fromBytes`
+  // methods for readers and writers to access this underlying array of bytes.
+  private final byte[] value;
+
+  // We make the constructor private. We should use `fromBytes` to create new instances.
+  private GeographyVal(byte[] value) {
+    this.value = value;
+  }
+
+  public byte[] getBytes() {
+    return value;
+  }
+
+  public static GeographyVal fromBytes(byte[] bytes) {
+    if (bytes == null) {
+      return null;
+    } else {
+      return new GeographyVal(bytes);
+    }
+  }
+
+  // Comparison is not yet supported for GEOGRAPHY.
+  public int compareTo(GeographyVal g) {
+    throw new UnsupportedOperationException();
+  }
+}

common/unsafe/src/main/java/org/apache/spark/unsafe/types/GeometryVal.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.unsafe.types;
+
+import java.io.Serializable;
+
+// This class represents the physical type for the GEOMETRY data type.
+public final class GeometryVal implements Comparable<GeometryVal>, Serializable {
+
+  // The GEOMETRY type is implemented as a byte array. We provide `getBytes` and `fromBytes`
+  // methods for readers and writers to access this underlying array of bytes.
+  private final byte[] value;
+
+  // We make the constructor private. We should use `fromBytes` to create new instances.
+  private GeometryVal(byte[] value) {
+    this.value = value;
+  }
+
+  public byte[] getBytes() {
+    return value;
+  }
+
+  public static GeometryVal fromBytes(byte[] bytes) {
+    if (bytes == null) {
+      return null;
+    } else {
+      return new GeometryVal(bytes);
+    }
+  }
+
+  // Comparison is not yet supported for GEOMETRY.
+  public int compareTo(GeometryVal g) {
+    throw new UnsupportedOperationException();
+  }
+}

common/unsafe/src/test/java/org/apache/spark/unsafe/types/GeographyValSuite.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.unsafe.types;
+
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class GeographyValSuite {
+
+  @Test
+  public void roundTripBytes() {
+    // A simple byte array to test the round trip (`fromBytes` -> `getBytes`).
+    byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6 };
+    GeographyVal geographyVal = GeographyVal.fromBytes(bytes);
+    assertNotNull(geographyVal);
+    assertArrayEquals(bytes, geographyVal.getBytes());
+  }
+
+  @Test
+  public void roundNullHandling() {
+    // A simple null byte array to test null handling for GEOGRAPHY.
+    byte[] bytes = null;
+    GeographyVal geographyVal = GeographyVal.fromBytes(bytes);
+    assertNull(geographyVal);
+  }
+
+  @Test
+  public void testCompareTo() {
+    // Comparison is not yet supported for GEOGRAPHY.
+    byte[] bytes1 = new byte[] { 1, 2, 3 };
+    byte[] bytes2 = new byte[] { 4, 5, 6 };
+    GeographyVal geographyVal1 = GeographyVal.fromBytes(bytes1);
+    GeographyVal geographyVal2 = GeographyVal.fromBytes(bytes2);
+    try {
+      geographyVal1.compareTo(geographyVal2);
+    } catch (UnsupportedOperationException e) {
+      assert(e.toString().equals("java.lang.UnsupportedOperationException"));
+    }
+  }
+}

common/unsafe/src/test/java/org/apache/spark/unsafe/types/GeometryValSuite.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.unsafe.types;
+
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class GeometryValSuite {
+
+  @Test
+  public void roundTripBytes() {
+    // A simple byte array to test the round trip (`fromBytes` -> `getBytes`).
+    byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6 };
+    GeometryVal geometryVal = GeometryVal.fromBytes(bytes);
+    assertNotNull(geometryVal);
+    assertArrayEquals(bytes, geometryVal.getBytes());
+  }
+
+  @Test
+  public void roundNullHandling() {
+    // A simple null byte array to test null handling for GEOMETRY.
+    byte[] bytes = null;
+    GeometryVal geometryVal = GeometryVal.fromBytes(bytes);
+    assertNull(geometryVal);
+  }
+
+  @Test
+  public void testCompareTo() {
+    // Comparison is not yet supported for GEOMETRY.
+    byte[] bytes1 = new byte[] { 1, 2, 3 };
+    byte[] bytes2 = new byte[] { 4, 5, 6 };
+    GeometryVal geometryVal1 = GeometryVal.fromBytes(bytes1);
+    GeometryVal geometryVal2 = GeometryVal.fromBytes(bytes2);
+    try {
+      geometryVal1.compareTo(geometryVal2);
+    } catch (UnsupportedOperationException e) {
+      assert(e.toString().equals("java.lang.UnsupportedOperationException"));
+    }
+  }
+}

core/src/main/scala/org/apache/spark/util/Utils.scala

@@ -3138,7 +3138,19 @@ private[spark] object Utils
   /**
    * Return whether we are using G1GC or not
    */
-  lazy val isG1GC: Boolean = {
+  lazy val isG1GC: Boolean = checkUseGC("UseG1GC")
+
+  /**
+   * Return whether we are using ZGC or not
+   */
+  lazy val isZGC: Boolean = checkUseGC("UseZGC")
+
+  /**
+   * Return whether we are using ShenandoahGC or not
+   */
+  lazy val isShenandoahGC: Boolean = checkUseGC("UseShenandoahGC")
+
+  def checkUseGC(useGCObjectStr: String): Boolean = {
     Try {
       val clazz = Utils.classForName("com.sun.management.HotSpotDiagnosticMXBean")
         .asInstanceOf[Class[_ <: PlatformManagedObject]]
@@ -3147,9 +3159,9 @@ private[spark] object Utils
       val vmOptionMethod = clazz.getMethod("getVMOption", classOf[String])
       val valueMethod = vmOptionClazz.getMethod("getValue")
 
-      val useG1GCObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC")
-      val useG1GC = valueMethod.invoke(useG1GCObject).asInstanceOf[String]
-      "true".equals(useG1GC)
+      val useGCObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, useGCObjectStr)
+      val useGC = valueMethod.invoke(useGCObject).asInstanceOf[String]
+      "true".equals(useGC)
     }.getOrElse(false)
   }
 

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -198,48 +198,48 @@ metrics-jmx/4.2.33//metrics-jmx-4.2.33.jar
 metrics-json/4.2.33//metrics-json-4.2.33.jar
 metrics-jvm/4.2.33//metrics-jvm-4.2.33.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.2.6.Final//netty-all-4.2.6.Final.jar
-netty-buffer/4.2.6.Final//netty-buffer-4.2.6.Final.jar
-netty-codec-base/4.2.6.Final//netty-codec-base-4.2.6.Final.jar
-netty-codec-classes-quic/4.2.6.Final//netty-codec-classes-quic-4.2.6.Final.jar
-netty-codec-compression/4.2.6.Final//netty-codec-compression-4.2.6.Final.jar
-netty-codec-dns/4.2.6.Final//netty-codec-dns-4.2.6.Final.jar
-netty-codec-http/4.2.6.Final//netty-codec-http-4.2.6.Final.jar
-netty-codec-http2/4.2.6.Final//netty-codec-http2-4.2.6.Final.jar
-netty-codec-http3/4.2.6.Final//netty-codec-http3-4.2.6.Final.jar
-netty-codec-marshalling/4.2.6.Final//netty-codec-marshalling-4.2.6.Final.jar
-netty-codec-native-quic/4.2.6.Final/linux-aarch_64/netty-codec-native-quic-4.2.6.Final-linux-aarch_64.jar
-netty-codec-native-quic/4.2.6.Final/linux-x86_64/netty-codec-native-quic-4.2.6.Final-linux-x86_64.jar
-netty-codec-native-quic/4.2.6.Final/osx-aarch_64/netty-codec-native-quic-4.2.6.Final-osx-aarch_64.jar
-netty-codec-native-quic/4.2.6.Final/osx-x86_64/netty-codec-native-quic-4.2.6.Final-osx-x86_64.jar
-netty-codec-native-quic/4.2.6.Final/windows-x86_64/netty-codec-native-quic-4.2.6.Final-windows-x86_64.jar
-netty-codec-protobuf/4.2.6.Final//netty-codec-protobuf-4.2.6.Final.jar
-netty-codec-socks/4.2.6.Final//netty-codec-socks-4.2.6.Final.jar
-netty-codec/4.2.6.Final//netty-codec-4.2.6.Final.jar
-netty-common/4.2.6.Final//netty-common-4.2.6.Final.jar
-netty-handler-proxy/4.2.6.Final//netty-handler-proxy-4.2.6.Final.jar
-netty-handler/4.2.6.Final//netty-handler-4.2.6.Final.jar
-netty-resolver-dns/4.2.6.Final//netty-resolver-dns-4.2.6.Final.jar
-netty-resolver/4.2.6.Final//netty-resolver-4.2.6.Final.jar
+netty-all/4.2.7.Final//netty-all-4.2.7.Final.jar
+netty-buffer/4.2.7.Final//netty-buffer-4.2.7.Final.jar
+netty-codec-base/4.2.7.Final//netty-codec-base-4.2.7.Final.jar
+netty-codec-classes-quic/4.2.7.Final//netty-codec-classes-quic-4.2.7.Final.jar
+netty-codec-compression/4.2.7.Final//netty-codec-compression-4.2.7.Final.jar
+netty-codec-dns/4.2.7.Final//netty-codec-dns-4.2.7.Final.jar
+netty-codec-http/4.2.7.Final//netty-codec-http-4.2.7.Final.jar
+netty-codec-http2/4.2.7.Final//netty-codec-http2-4.2.7.Final.jar
+netty-codec-http3/4.2.7.Final//netty-codec-http3-4.2.7.Final.jar
+netty-codec-marshalling/4.2.7.Final//netty-codec-marshalling-4.2.7.Final.jar
+netty-codec-native-quic/4.2.7.Final/linux-aarch_64/netty-codec-native-quic-4.2.7.Final-linux-aarch_64.jar
+netty-codec-native-quic/4.2.7.Final/linux-x86_64/netty-codec-native-quic-4.2.7.Final-linux-x86_64.jar
+netty-codec-native-quic/4.2.7.Final/osx-aarch_64/netty-codec-native-quic-4.2.7.Final-osx-aarch_64.jar
+netty-codec-native-quic/4.2.7.Final/osx-x86_64/netty-codec-native-quic-4.2.7.Final-osx-x86_64.jar
+netty-codec-native-quic/4.2.7.Final/windows-x86_64/netty-codec-native-quic-4.2.7.Final-windows-x86_64.jar
+netty-codec-protobuf/4.2.7.Final//netty-codec-protobuf-4.2.7.Final.jar
+netty-codec-socks/4.2.7.Final//netty-codec-socks-4.2.7.Final.jar
+netty-codec/4.2.7.Final//netty-codec-4.2.7.Final.jar
+netty-common/4.2.7.Final//netty-common-4.2.7.Final.jar
+netty-handler-proxy/4.2.7.Final//netty-handler-proxy-4.2.7.Final.jar
+netty-handler/4.2.7.Final//netty-handler-4.2.7.Final.jar
+netty-resolver-dns/4.2.7.Final//netty-resolver-dns-4.2.7.Final.jar
+netty-resolver/4.2.7.Final//netty-resolver-4.2.7.Final.jar
 netty-tcnative-boringssl-static/2.0.74.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar
 netty-tcnative-boringssl-static/2.0.74.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar
 netty-tcnative-classes/2.0.74.Final//netty-tcnative-classes-2.0.74.Final.jar
-netty-transport-classes-epoll/4.2.6.Final//netty-transport-classes-epoll-4.2.6.Final.jar
-netty-transport-classes-io_uring/4.2.6.Final//netty-transport-classes-io_uring-4.2.6.Final.jar
-netty-transport-classes-kqueue/4.2.6.Final//netty-transport-classes-kqueue-4.2.6.Final.jar
-netty-transport-native-epoll/4.2.6.Final/linux-aarch_64/netty-transport-native-epoll-4.2.6.Final-linux-aarch_64.jar
-netty-transport-native-epoll/4.2.6.Final/linux-riscv64/netty-transport-native-epoll-4.2.6.Final-linux-riscv64.jar
-netty-transport-native-epoll/4.2.6.Final/linux-x86_64/netty-transport-native-epoll-4.2.6.Final-linux-x86_64.jar
-netty-transport-native-io_uring/4.2.6.Final/linux-aarch_64/netty-transport-native-io_uring-4.2.6.Final-linux-aarch_64.jar
-netty-transport-native-io_uring/4.2.6.Final/linux-riscv64/netty-transport-native-io_uring-4.2.6.Final-linux-riscv64.jar
-netty-transport-native-io_uring/4.2.6.Final/linux-x86_64/netty-transport-native-io_uring-4.2.6.Final-linux-x86_64.jar
-netty-transport-native-kqueue/4.2.6.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.6.Final-osx-aarch_64.jar
-netty-transport-native-kqueue/4.2.6.Final/osx-x86_64/netty-transport-native-kqueue-4.2.6.Final-osx-x86_64.jar
-netty-transport-native-unix-common/4.2.6.Final//netty-transport-native-unix-common-4.2.6.Final.jar
-netty-transport/4.2.6.Final//netty-transport-4.2.6.Final.jar
+netty-transport-classes-epoll/4.2.7.Final//netty-transport-classes-epoll-4.2.7.Final.jar
+netty-transport-classes-io_uring/4.2.7.Final//netty-transport-classes-io_uring-4.2.7.Final.jar
+netty-transport-classes-kqueue/4.2.7.Final//netty-transport-classes-kqueue-4.2.7.Final.jar
+netty-transport-native-epoll/4.2.7.Final/linux-aarch_64/netty-transport-native-epoll-4.2.7.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.2.7.Final/linux-riscv64/netty-transport-native-epoll-4.2.7.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.2.7.Final/linux-x86_64/netty-transport-native-epoll-4.2.7.Final-linux-x86_64.jar
+netty-transport-native-io_uring/4.2.7.Final/linux-aarch_64/netty-transport-native-io_uring-4.2.7.Final-linux-aarch_64.jar
+netty-transport-native-io_uring/4.2.7.Final/linux-riscv64/netty-transport-native-io_uring-4.2.7.Final-linux-riscv64.jar
+netty-transport-native-io_uring/4.2.7.Final/linux-x86_64/netty-transport-native-io_uring-4.2.7.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.2.7.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.7.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.2.7.Final/osx-x86_64/netty-transport-native-kqueue-4.2.7.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.2.7.Final//netty-transport-native-unix-common-4.2.7.Final.jar
+netty-transport/4.2.7.Final//netty-transport-4.2.7.Final.jar
 objenesis/3.4//objenesis-3.4.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.17.6//okio-1.17.6.jar

pom.xml

@@ -217,7 +217,7 @@
     <bouncycastle.version>1.82</bouncycastle.version>
     <tink.version>1.16.0</tink.version>
     <datasketches.version>6.2.0</datasketches.version>
-    <netty.version>4.2.6.Final</netty.version>
+    <netty.version>4.2.7.Final</netty.version>
     <netty-tcnative.version>2.0.74.Final</netty-tcnative.version>
     <icu4j.version>77.1</icu4j.version>
     <junit.version>6.0.0</junit.version>