Upgrade parquet to 1.12.0-palantir.1 (apache-spark-on-k8s#464)

Author: robert3005

dev/deps/spark-deps-hadoop-palantir

@@ -13,6 +13,7 @@ arpack_combined_all-0.1.jar
 arrow-format-0.10.0.jar
 arrow-memory-0.10.0.jar
 arrow-vector-0.10.0.jar
+audience-annotations-0.7.0.jar
 automaton-1.11-8.jar
 avro-1.8.2.jar
 avro-ipc-1.8.2.jar
@@ -170,12 +171,12 @@ orc-shims-1.5.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
 paranamer-2.8.jar
-parquet-column-1.10.1-palantir.3.jar
-parquet-common-1.10.1-palantir.3.jar
-parquet-encoding-1.10.1-palantir.3.jar
-parquet-format-2.4.0.jar
-parquet-hadoop-1.10.1-palantir.3.jar
-parquet-jackson-1.10.1-palantir.3.jar
+parquet-column-1.12.0-palantir.1.jar
+parquet-common-1.12.0-palantir.1.jar
+parquet-encoding-1.12.0-palantir.1.jar
+parquet-format-structures-1.12.0-palantir.1.jar
+parquet-hadoop-1.12.0-palantir.1.jar
+parquet-jackson-1.12.0-palantir.1.jar
 protobuf-java-2.5.0.jar
 py4j-0.10.8.1.jar
 pyrolite-4.13.jar

pom.xml

@@ -130,10 +130,11 @@
     <!-- Version used for internal directory structure -->
     <hive.version.short>1.2.1</hive.version.short>
     <derby.version>10.12.1.1</derby.version>
-    <parquet.version>1.10.1-palantir.3</parquet.version>
-      <okhttp3.version>3.11.0</okhttp3.version>
-      <okhttp.version>2.7.5</okhttp.version>
-      <okio.version>1.15.0</okio.version>
+    <parquet.version>1.12.0-palantir.1</parquet.version>
+    <okhttp3.version>3.11.0</okhttp3.version>
+    <okhttp.version>2.7.5</okhttp.version>
+    <okio.version>1.15.0</okio.version>
+    <jetty.version>9.4.12.v20180830</jetty.version>
     <orc.version>1.5.3</orc.version>
     <orc.classifier>nohive</orc.classifier>
     <hive.parquet.version>1.6.0</hive.parquet.version>

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala

@@ -618,8 +618,8 @@ private[parquet] class ParquetFilters(
               }
 
               override def keep(value: Binary): Boolean = {
-                UTF8String.fromBytes(value.getBytes).startsWith(
-                  UTF8String.fromBytes(strToBinary.getBytes))
+                value != null && UTF8String.fromBytes(value.getBytes)
+                  .startsWith(UTF8String.fromBytes(strToBinary.getBytes))
               }
             }
           )

sql/core/src/test/resources/sql-tests/results/describe-part-after-analyze.sql.out

@@ -93,7 +93,7 @@ Partition Values    	[ds=2017-08-01, hr=10]
 Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10	                    
 Created Time [not included in comparison]
 Last Access [not included in comparison]
-Partition Statistics	1195 bytes, 3 rows  	                    
+Partition Statistics	1264 bytes, 3 rows  	                    
 
 # Storage Information	                    	                    
 Location [not included in comparison]sql/core/spark-warehouse/t
@@ -128,7 +128,7 @@ Partition Values    	[ds=2017-08-01, hr=10]
 Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10	                    
 Created Time [not included in comparison]
 Last Access [not included in comparison]
-Partition Statistics	1195 bytes, 3 rows  	                    
+Partition Statistics	1264 bytes, 3 rows  	                    
 
 # Storage Information	                    	                    
 Location [not included in comparison]sql/core/spark-warehouse/t
@@ -155,7 +155,7 @@ Partition Values    	[ds=2017-08-01, hr=11]
 Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=11	                    
 Created Time [not included in comparison]
 Last Access [not included in comparison]
-Partition Statistics	1208 bytes, 4 rows  	                    
+Partition Statistics	1278 bytes, 4 rows  	                    
 
 # Storage Information	                    	                    
 Location [not included in comparison]sql/core/spark-warehouse/t
@@ -190,7 +190,7 @@ Partition Values    	[ds=2017-08-01, hr=10]
 Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=10	                    
 Created Time [not included in comparison]
 Last Access [not included in comparison]
-Partition Statistics	1195 bytes, 3 rows  	                    
+Partition Statistics	1264 bytes, 3 rows  	                    
 
 # Storage Information	                    	                    
 Location [not included in comparison]sql/core/spark-warehouse/t
@@ -217,7 +217,7 @@ Partition Values    	[ds=2017-08-01, hr=11]
 Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-08-01/hr=11	                    
 Created Time [not included in comparison]
 Last Access [not included in comparison]
-Partition Statistics	1208 bytes, 4 rows  	                    
+Partition Statistics	1278 bytes, 4 rows  	                    
 
 # Storage Information	                    	                    
 Location [not included in comparison]sql/core/spark-warehouse/t
@@ -244,7 +244,7 @@ Partition Values    	[ds=2017-09-01, hr=5]
 Location [not included in comparison]sql/core/spark-warehouse/t/ds=2017-09-01/hr=5	                    
 Created Time [not included in comparison]
 Last Access [not included in comparison]
-Partition Statistics	1182 bytes, 2 rows  	                    
+Partition Statistics	1250 bytes, 2 rows  	                    
 
 # Storage Information	                    	                    
 Location [not included in comparison]sql/core/spark-warehouse/t

sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala

@@ -506,7 +506,7 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSQLContext {
             case plan: InMemoryRelation => plan
           }.head
           // InMemoryRelation's stats is file size before the underlying RDD is materialized
-          assert(inMemoryRelation.computeStats().sizeInBytes === 822)
+          assert(inMemoryRelation.computeStats().sizeInBytes === 848)
 
           // InMemoryRelation's stats is updated after materializing RDD
           dfFromFile.collect()
@@ -519,7 +519,7 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSQLContext {
 
           // Even CBO enabled, InMemoryRelation's stats keeps as the file size before table's stats
           // is calculated
-          assert(inMemoryRelation2.computeStats().sizeInBytes === 822)
+          assert(inMemoryRelation2.computeStats().sizeInBytes === 848)
 
           // InMemoryRelation's stats should be updated after calculating stats of the table
           // clear cache to simulate a fresh environment

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelationSuite.scala

@@ -45,7 +45,7 @@ class HadoopFsRelationSuite extends QueryTest with SharedSQLContext {
     import testImplicits._
     Seq(1.0, 0.5).foreach { compressionFactor =>
       withSQLConf("spark.sql.sources.fileCompressionFactor" -> compressionFactor.toString,
-        "spark.sql.autoBroadcastJoinThreshold" -> "411") {
+        "spark.sql.autoBroadcastJoinThreshold" -> "424") {
         withTempPath { workDir =>
           // the file size is 740 bytes
           val workDirPath = workDir.getAbsolutePath