Migrate parquet-variant-compute to Rust 2024 (#8511)

# Which issue does this PR close?

- Contribute to #6827

# Rationale for this change

Splitting up #8227.

# What changes are included in this PR?

Migrate `parquet-variant-compute` to Rust 2024

# Are these changes tested?

CI

# Are there any user-facing changes?

Yes

Author: mbrobbel

parquet-variant-compute/Cargo.toml

@@ -26,7 +26,7 @@ homepage = { workspace = true }
 repository = { workspace = true }
 authors = { workspace = true }
 keywords = ["arrow", "parquet", "variant"]
-edition = { workspace = true }
+edition = "2024"
 rust-version = { workspace = true }
 
 

parquet-variant-compute/benches/variant_kernels.rs

@@ -17,14 +17,14 @@
 
 use arrow::array::{Array, ArrayRef, StringArray};
 use arrow::util::test_util::seedable_rng;
-use criterion::{criterion_group, criterion_main, Criterion};
+use criterion::{Criterion, criterion_group, criterion_main};
 use parquet_variant::{Variant, VariantBuilder};
-use parquet_variant_compute::variant_get::{variant_get, GetOptions};
-use parquet_variant_compute::{json_to_variant, VariantArray, VariantArrayBuilder};
-use rand::distr::Alphanumeric;
-use rand::rngs::StdRng;
+use parquet_variant_compute::variant_get::{GetOptions, variant_get};
+use parquet_variant_compute::{VariantArray, VariantArrayBuilder, json_to_variant};
 use rand::Rng;
 use rand::SeedableRng;
+use rand::distr::Alphanumeric;
+use rand::rngs::StdRng;
 use std::fmt::Write;
 use std::sync::Arc;
 fn benchmark_batch_json_string_to_variant(c: &mut Criterion) {

parquet-variant-compute/src/arrow_to_variant.rs

@@ -15,25 +15,25 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::type_conversion::{decimal_to_variant_decimal, CastOptions};
+use crate::type_conversion::{CastOptions, decimal_to_variant_decimal};
 use arrow::array::{
     Array, AsArray, FixedSizeListArray, GenericBinaryArray, GenericListArray, GenericListViewArray,
     GenericStringArray, OffsetSizeTrait, PrimitiveArray,
 };
 use arrow::compute::kernels::cast;
 use arrow::datatypes::{
     ArrowNativeType, ArrowPrimitiveType, ArrowTemporalType, ArrowTimestampType, Date32Type,
-    Date64Type, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type,
+    Date64Type, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
     RunEndIndexType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
     Time64NanosecondType, TimestampMicrosecondType, TimestampMillisecondType,
-    TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
+    TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
 };
 use arrow::temporal_conversions::{as_date, as_datetime, as_time};
 use arrow_schema::{ArrowError, DataType, TimeUnit};
 use chrono::{DateTime, TimeZone, Utc};
 use parquet_variant::{
-    ObjectFieldBuilder, Variant, VariantBuilderExt, VariantDecimal16, VariantDecimal4,
-    VariantDecimal8,
+    ObjectFieldBuilder, Variant, VariantBuilderExt, VariantDecimal4, VariantDecimal8,
+    VariantDecimal16,
 };
 use std::collections::HashMap;
 use std::ops::Range;
@@ -213,7 +213,7 @@ pub(crate) fn make_arrow_to_variant_row_builder<'a>(
                 _ => {
                     return Err(ArrowError::CastError(format!(
                         "Unsupported Time32 unit: {time_unit:?}"
-                    )))
+                    )));
                 }
             },
             DataType::Time64(time_unit) => match time_unit {
@@ -226,15 +226,15 @@ pub(crate) fn make_arrow_to_variant_row_builder<'a>(
                 _ => {
                     return Err(ArrowError::CastError(format!(
                         "Unsupported Time64 unit: {time_unit:?}"
-                    )))
+                    )));
                 }
             },
             DataType::Duration(_) | DataType::Interval(_) => {
                 return Err(ArrowError::InvalidArgumentError(
                     "Casting duration/interval types to Variant is not supported. \
                     The Variant format does not define duration/interval types."
                         .to_string(),
-                ))
+                ));
             }
             DataType::Binary => Binary(BinaryArrowToVariantBuilder::new(array)),
             DataType::LargeBinary => LargeBinary(BinaryArrowToVariantBuilder::new(array)),

parquet-variant-compute/src/cast_to_variant.rs

@@ -84,33 +84,34 @@ pub fn cast_to_variant(input: &dyn Array) -> Result<VariantArray, ArrowError> {
 mod tests {
     use super::*;
     use arrow::array::{
-        ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal128Array,
-        Decimal256Array, Decimal32Array, Decimal64Array, DictionaryArray, DurationMicrosecondArray,
-        DurationMillisecondArray, DurationNanosecondArray, DurationSecondArray,
-        FixedSizeBinaryBuilder, FixedSizeListBuilder, Float16Array, Float32Array, Float64Array,
-        GenericByteBuilder, GenericByteViewBuilder, Int16Array, Int32Array, Int64Array, Int8Array,
-        IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeListArray,
-        LargeListViewBuilder, LargeStringArray, ListArray, ListViewBuilder, MapArray, NullArray,
-        StringArray, StringRunBuilder, StringViewArray, StructArray, Time32MillisecondArray,
-        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
-        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
-        TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array, UnionArray,
+        ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal32Array,
+        Decimal64Array, Decimal128Array, Decimal256Array, DictionaryArray,
+        DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray,
+        DurationSecondArray, FixedSizeBinaryBuilder, FixedSizeListBuilder, Float16Array,
+        Float32Array, Float64Array, GenericByteBuilder, GenericByteViewBuilder, Int8Array,
+        Int16Array, Int32Array, Int64Array, IntervalDayTimeArray, IntervalMonthDayNanoArray,
+        IntervalYearMonthArray, LargeListArray, LargeListViewBuilder, LargeStringArray, ListArray,
+        ListViewBuilder, MapArray, NullArray, StringArray, StringRunBuilder, StringViewArray,
+        StructArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,
+        Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,
+        TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array,
+        UInt64Array, UnionArray,
     };
     use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
     use arrow::datatypes::{
-        i256, BinaryType, BinaryViewType, Date32Type, Date64Type, Int32Type, Int64Type, Int8Type,
-        IntervalDayTime, IntervalMonthDayNano, LargeBinaryType,
+        BinaryType, BinaryViewType, Date32Type, Date64Type, Int8Type, Int32Type, Int64Type,
+        IntervalDayTime, IntervalMonthDayNano, LargeBinaryType, i256,
     };
     use arrow::temporal_conversions::timestamp_s_to_datetime;
-    use arrow_schema::{DataType, Field, Fields, UnionFields};
     use arrow_schema::{
-        DECIMAL128_MAX_PRECISION, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION,
+        DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION,
     };
+    use arrow_schema::{DataType, Field, Fields, UnionFields};
     use chrono::{DateTime, NaiveDate, NaiveTime};
     use half::f16;
     use parquet_variant::{
-        Variant, VariantBuilder, VariantBuilderExt, VariantDecimal16, VariantDecimal4,
-        VariantDecimal8,
+        Variant, VariantBuilder, VariantBuilderExt, VariantDecimal4, VariantDecimal8,
+        VariantDecimal16,
     };
     use std::{sync::Arc, vec};
 

parquet-variant-compute/src/shred_variant.rs

@@ -19,7 +19,7 @@
 
 use crate::variant_array::{ShreddedVariantFieldArray, StructArrayBuilder};
 use crate::variant_to_arrow::{
-    make_primitive_variant_to_arrow_row_builder, PrimitiveVariantToArrowRowBuilder,
+    PrimitiveVariantToArrowRowBuilder, make_primitive_variant_to_arrow_row_builder,
 };
 use crate::{VariantArray, VariantValueArrayBuilder};
 use arrow::array::{ArrayRef, BinaryViewArray, NullBufferBuilder};

parquet-variant-compute/src/to_json.rs

@@ -49,7 +49,7 @@ pub fn variant_to_json(input: &ArrayRef) -> Result<StringArray, ArrowError> {
         _ => {
             return Err(ArrowError::CastError(
                 "Expected StructArray with known fields".into(),
-            ))
+            ));
         }
     }
 

parquet-variant-compute/src/unshred_variant.rs

@@ -25,8 +25,8 @@ use arrow::array::{
 };
 use arrow::buffer::NullBuffer;
 use arrow::datatypes::{
-    ArrowPrimitiveType, DataType, Date32Type, Float32Type, Float64Type, Int16Type, Int32Type,
-    Int64Type, Int8Type, Time64MicrosecondType, TimeUnit, TimestampMicrosecondType,
+    ArrowPrimitiveType, DataType, Date32Type, Float32Type, Float64Type, Int8Type, Int16Type,
+    Int32Type, Int64Type, Time64MicrosecondType, TimeUnit, TimestampMicrosecondType,
     TimestampNanosecondType,
 };
 use arrow::error::{ArrowError, Result};

parquet-variant-compute/src/variant_array.rs

@@ -22,7 +22,7 @@ use arrow::array::{Array, ArrayRef, AsArray, BinaryViewArray, StructArray};
 use arrow::buffer::NullBuffer;
 use arrow::compute::cast;
 use arrow::datatypes::{
-    Date32Type, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type,
+    Date32Type, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
     TimestampMicrosecondType, TimestampNanosecondType,
 };
 use arrow_schema::extension::ExtensionType;

parquet-variant-compute/src/variant_get.rs

@@ -23,9 +23,9 @@ use arrow::{
 use arrow_schema::{ArrowError, DataType, FieldRef};
 use parquet_variant::{VariantPath, VariantPathElement};
 
+use crate::VariantArray;
 use crate::variant_array::BorrowedShreddingState;
 use crate::variant_to_arrow::make_variant_to_arrow_row_builder;
-use crate::VariantArray;
 
 use arrow::array::AsArray;
 use std::sync::Arc;
@@ -295,20 +295,20 @@ impl<'a> GetOptions<'a> {
 mod test {
     use std::sync::Arc;
 
-    use super::{variant_get, GetOptions};
+    use super::{GetOptions, variant_get};
+    use crate::VariantArray;
     use crate::json_to_variant;
     use crate::variant_array::{ShreddedVariantFieldArray, StructArrayBuilder};
-    use crate::VariantArray;
     use arrow::array::{
         Array, ArrayRef, AsArray, BinaryViewArray, Date32Array, Float32Array, Float64Array,
-        Int16Array, Int32Array, Int64Array, Int8Array, StringArray, StructArray,
+        Int8Array, Int16Array, Int32Array, Int64Array, StringArray, StructArray,
     };
     use arrow::buffer::NullBuffer;
     use arrow::compute::CastOptions;
     use arrow::datatypes::DataType::{Int16, Int32, Int64};
     use arrow_schema::{DataType, Field, FieldRef, Fields};
     use chrono::DateTime;
-    use parquet_variant::{Variant, VariantPath, EMPTY_VARIANT_METADATA_BYTES};
+    use parquet_variant::{EMPTY_VARIANT_METADATA_BYTES, Variant, VariantPath};
 
     fn single_variant_get_test(input_json: &str, path: VariantPath, expected_json: &str) {
         // Create input array from JSON string
@@ -602,7 +602,10 @@ mod test {
 
         let err = variant_get(&array, options).unwrap_err();
         // TODO make this error message nicer (not Debug format)
-        assert_eq!(err.to_string(), "Cast error: Failed to extract primitive of type Int32 from variant ShortString(ShortString(\"n/a\")) at path VariantPath([])");
+        assert_eq!(
+            err.to_string(),
+            "Cast error: Failed to extract primitive of type Int32 from variant ShortString(ShortString(\"n/a\")) at path VariantPath([])"
+        );
     }
 
     /// Perfect Shredding: extract the typed value as a VariantArray
@@ -1926,9 +1929,11 @@ mod test {
         assert!(result.is_err());
         let error = result.unwrap_err();
         assert!(matches!(error, ArrowError::CastError(_)));
-        assert!(error
-            .to_string()
-            .contains("Cannot access field 'nonexistent_field' on non-struct type"));
+        assert!(
+            error
+                .to_string()
+                .contains("Cannot access field 'nonexistent_field' on non-struct type")
+        );
     }
 
     #[test]