diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0ff7b3cd9f892..f0a293a5e0593 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -25,7 +25,7 @@ on: java: required: false type: string - default: 17 + default: 21 branch: description: Branch to run the build against required: false diff --git a/LICENSE-binary b/LICENSE-binary index 2892c4b0ecce8..5dfe37a13b0a4 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -392,7 +392,7 @@ org.json4s:json4s-jackson_2.13 org.json4s:json4s-scalap_2.13 org.lz4:lz4-java org.objenesis:objenesis -org.roaringbitmap:RoaringBitmap +com.github.RoaringBitmap.RoaringBitmap:roaringbitmap org.rocksdb:rocksdbjni org.scala-lang:scala-compiler org.scala-lang:scala-library diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 2f2be88ac9dd1..56db3b030906d 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -136,8 +136,8 @@ tink - org.roaringbitmap - RoaringBitmap + com.github.RoaringBitmap.RoaringBitmap + roaringbitmap diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index adfc55d28c357..d5969c939e5d5 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -63,8 +63,8 @@ guava - org.roaringbitmap - RoaringBitmap + com.github.RoaringBitmap.RoaringBitmap + roaringbitmap diff --git a/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt b/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt index 123a40fad3e62..1d4a189863b95 100644 --- a/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt +++ b/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt @@ -2,12 +2,12 @@ MapStatuses Convert Benchmark ================================================================================================ -OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.8.0-1017-azure +OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure AMD EPYC 7763 64-Core Processor MapStatuses Convert: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Num Maps: 50000 Fetch partitions:500 716 730 14 0.0 715747604.0 1.0X -Num Maps: 50000 Fetch partitions:1000 1592 1619 29 0.0 1591519021.0 0.4X -Num Maps: 50000 Fetch partitions:1500 2500 2507 7 0.0 2499934291.0 0.3X +Num Maps: 50000 Fetch partitions:500 726 733 6 0.0 726407701.0 1.0X +Num Maps: 50000 Fetch partitions:1000 1644 1659 14 0.0 1644292738.0 0.4X +Num Maps: 50000 Fetch partitions:1500 2482 2514 31 0.0 2482389083.0 0.3X diff --git a/core/benchmarks/MapStatusesConvertBenchmark-results.txt b/core/benchmarks/MapStatusesConvertBenchmark-results.txt index ef390204bb0af..f619251ec9548 100644 --- a/core/benchmarks/MapStatusesConvertBenchmark-results.txt +++ b/core/benchmarks/MapStatusesConvertBenchmark-results.txt @@ -2,12 +2,12 @@ MapStatuses Convert Benchmark ================================================================================================ -OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.8.0-1017-azure +OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure AMD EPYC 7763 64-Core Processor MapStatuses Convert: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Num Maps: 50000 Fetch partitions:500 612 614 3 0.0 611543498.0 1.0X -Num Maps: 50000 Fetch partitions:1000 1389 1398 15 0.0 1388971632.0 0.4X -Num Maps: 50000 Fetch partitions:1500 2178 2222 39 0.0 2177711722.0 0.3X +Num Maps: 50000 Fetch partitions:500 718 727 13 0.0 717856848.0 1.0X +Num Maps: 50000 Fetch partitions:1000 1730 1751 25 0.0 1729819265.0 0.4X +Num Maps: 50000 Fetch partitions:1500 2662 2692 27 0.0 2662049238.0 0.3X diff --git a/core/pom.xml b/core/pom.xml index 85b6d2666585b..6e5043b79def8 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -240,8 +240,8 @@ zstd-jni - org.roaringbitmap - RoaringBitmap + com.github.RoaringBitmap.RoaringBitmap + roaringbitmap org.scala-lang.modules diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 88ac18fdde384..c389b69947d34 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -1,7 +1,6 @@ HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar -RoaringBitmap/1.3.0//RoaringBitmap-1.3.0.jar ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/2.0.2//aircompressor-2.0.2.jar @@ -252,6 +251,7 @@ parquet-jackson/1.15.0//parquet-jackson-1.15.0.jar pickle/1.5//pickle-1.5.jar py4j/0.10.9.9//py4j-0.10.9.9.jar remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar +roaringbitmap/1.3.19//roaringbitmap-1.3.19.jar rocksdbjni/9.8.4//rocksdbjni-9.8.4.jar scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar scala-compiler/2.13.15//scala-compiler-2.13.15.jar diff --git a/pom.xml b/pom.xml index 769dad270af77..38576e7ca41d2 100644 --- a/pom.xml +++ b/pom.xml @@ -356,6 +356,17 @@ false + + jitpack.io + JitPack + https://jitpack.io + + true + + + false + + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1d25215590af5..b4c02259ea6dc 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -277,6 +277,7 @@ object SparkBuild extends PomBuild { // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central. // See https://storage-download.googleapis.com/maven-central/index.html for more info. "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/", + "jitpack" at "https://jitpack.io", DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)