Migrate arrow-data to Rust 2024 (#8455)

mbrobbel · web-flow · commit 04cd6cf0ed67 · 2025-09-29T15:33:18.000-04:00
# Which issue does this PR close? - Contribute to #6827 # Rationale for this change Splitting up #8227. # What changes are included in this PR? Migrate `arrow-data` to Rust 2024 # Are these changes tested? CI # Are there any user-facing changes? Yes
diff --git a/arrow-data/Cargo.toml b/arrow-data/Cargo.toml
@@ -25,7 +25,7 @@ authors = { workspace = true }
 license = { workspace = true }
 keywords = { workspace = true }
 include = { workspace = true }
-edition = { workspace = true }
+edition = "2024"
 rust-version = { workspace = true }
 
 [lib]
diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs
@@ -21,7 +21,7 @@
 use crate::bit_iterator::BitSliceIterator;
 use arrow_buffer::buffer::{BooleanBuffer, NullBuffer};
 use arrow_buffer::{
-    bit_util, i256, ArrowNativeType, Buffer, IntervalDayTime, IntervalMonthDayNano, MutableBuffer,
+    ArrowNativeType, Buffer, IntervalDayTime, IntervalMonthDayNano, MutableBuffer, bit_util, i256,
 };
 use arrow_schema::{ArrowError, DataType, UnionMode};
 use std::mem;
@@ -281,7 +281,7 @@ impl ArrayData {
     ) -> Self {
         let mut skip_validation = UnsafeFlag::new();
         // SAFETY: caller responsible for ensuring data is valid
-        skip_validation.set(true);
+        unsafe { skip_validation.set(true) };
 
         ArrayDataBuilder {
             data_type,
@@ -476,21 +476,20 @@ impl ArrayData {
                     result += buffer_size;
                 }
                 BufferSpec::VariableWidth => {
-                    let buffer_len: usize;
-                    match self.data_type {
+                    let buffer_len = match self.data_type {
                         DataType::Utf8 | DataType::Binary => {
                             let offsets = self.typed_offsets::<i32>()?;
-                            buffer_len = (offsets[self.len] - offsets[0] ) as usize;
+                            (offsets[self.len] - offsets[0]) as usize
                         }
                         DataType::LargeUtf8 | DataType::LargeBinary => {
                             let offsets = self.typed_offsets::<i64>()?;
-                            buffer_len = (offsets[self.len] - offsets[0]) as usize;
+                            (offsets[self.len] - offsets[0]) as usize
                         }
                         _ => {
                             return Err(ArrowError::NotYetImplemented(format!(
-                            "Invalid data type for VariableWidth buffer. Expected Utf8, LargeUtf8, Binary or LargeBinary. Got {}",
-                            self.data_type
-                            )))
+                                "Invalid data type for VariableWidth buffer. Expected Utf8, LargeUtf8, Binary or LargeBinary. Got {}",
+                                self.data_type
+                            )));
                         }
                     };
                     result += buffer_len;
@@ -554,7 +553,7 @@ impl ArrayData {
         if let DataType::Struct(_) = self.data_type() {
             // Slice into children
             let new_offset = self.offset + offset;
-            let new_data = ArrayData {
+            ArrayData {
                 data_type: self.data_type().clone(),
                 len: length,
                 offset: new_offset,
@@ -566,9 +565,7 @@ impl ArrayData {
                     .map(|data| data.slice(offset, length))
                     .collect(),
                 nulls: self.nulls.as_ref().map(|x| x.slice(offset, length)),
-            };
-
-            new_data
+            }
         } else {
             let mut new_data = self.clone();
 
@@ -784,15 +781,19 @@ impl ArrayData {
                     if buffer.len() < min_buffer_size {
                         return Err(ArrowError::InvalidArgumentError(format!(
                             "Need at least {} bytes in buffers[{}] in array of type {:?}, but got {}",
-                            min_buffer_size, i, self.data_type, buffer.len()
+                            min_buffer_size,
+                            i,
+                            self.data_type,
+                            buffer.len()
                         )));
                     }
 
                     let align_offset = buffer.as_ptr().align_offset(*alignment);
                     if align_offset != 0 {
                         return Err(ArrowError::InvalidArgumentError(format!(
                             "Misaligned buffers[{i}] in array of type {:?}, offset from expected alignment of {alignment} by {}",
-                            self.data_type, align_offset.min(alignment - align_offset)
+                            self.data_type,
+                            align_offset.min(alignment - align_offset)
                         )));
                     }
                 }
@@ -806,7 +807,10 @@ impl ArrayData {
                     if buffer.len() < min_buffer_size {
                         return Err(ArrowError::InvalidArgumentError(format!(
                             "Need at least {} bytes for bitmap in buffers[{}] in array of type {:?}, but got {}",
-                            min_buffer_size, i, self.data_type, buffer.len()
+                            min_buffer_size,
+                            i,
+                            self.data_type,
+                            buffer.len()
                         )));
                     }
                 }
@@ -1058,7 +1062,11 @@ impl ArrayData {
                     if field_data.len < self.len {
                         return Err(ArrowError::InvalidArgumentError(format!(
                             "{} child array #{} for field {} has length smaller than expected for struct array ({} < {})",
-                            self.data_type, i, field.name(), field_data.len, self.len
+                            self.data_type,
+                            i,
+                            field.name(),
+                            field_data.len,
+                            self.len
                         )));
                     }
                 }
@@ -1090,7 +1098,9 @@ impl ArrayData {
                     if mode == &UnionMode::Sparse && field_data.len < (self.len + self.offset) {
                         return Err(ArrowError::InvalidArgumentError(format!(
                             "Sparse union child array #{} has length smaller than expected for union array ({} < {})",
-                            i, field_data.len, self.len + self.offset
+                            i,
+                            field_data.len,
+                            self.len + self.offset
                         )));
                     }
                 }
@@ -1282,7 +1292,7 @@ impl ArrayData {
                         "non-nullable child of type {} contains nulls not present in parent {}",
                         child.data_type, self.data_type
                     ))),
-                }
+                };
             }
         };
 
@@ -1988,6 +1998,7 @@ impl ArrayDataBuilder {
     ///
     /// Note: This is shorthand for
     /// ```rust
+    /// # #[expect(unsafe_op_in_unsafe_fn)]
     /// # let mut builder = arrow_data::ArrayDataBuilder::new(arrow_schema::DataType::Null);
     /// # let _ = unsafe {
     /// builder.skip_validation(true).build().unwrap()
@@ -1999,7 +2010,7 @@ impl ArrayDataBuilder {
     /// The same caveats as [`ArrayData::new_unchecked`]
     /// apply.
     pub unsafe fn build_unchecked(self) -> ArrayData {
-        self.skip_validation(true).build().unwrap()
+        unsafe { self.skip_validation(true) }.build().unwrap()
     }
 
     /// Creates an `ArrayData`, consuming `self`
@@ -2098,7 +2109,9 @@ impl ArrayDataBuilder {
     /// If validation is skipped, the buffers must form a valid Arrow array,
     /// otherwise undefined behavior will result
     pub unsafe fn skip_validation(mut self, skip_validation: bool) -> Self {
-        self.skip_validation.set(skip_validation);
+        unsafe {
+            self.skip_validation.set(skip_validation);
+        }
         self
     }
 }
diff --git a/arrow-data/src/decimal.rs b/arrow-data/src/decimal.rs
@@ -28,9 +28,9 @@ use arrow_buffer::i256;
 use arrow_schema::ArrowError;
 
 pub use arrow_schema::{
+    DECIMAL_DEFAULT_SCALE, DECIMAL32_DEFAULT_SCALE, DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE,
+    DECIMAL64_DEFAULT_SCALE, DECIMAL64_MAX_PRECISION, DECIMAL64_MAX_SCALE,
     DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE,
-    DECIMAL32_DEFAULT_SCALE, DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE, DECIMAL64_DEFAULT_SCALE,
-    DECIMAL64_MAX_PRECISION, DECIMAL64_MAX_SCALE, DECIMAL_DEFAULT_SCALE,
 };
 
 /// `MAX_DECIMAL256_FOR_EACH_PRECISION[p]` holds the maximum [`i256`] value that can
diff --git a/arrow-data/src/equal/boolean.rs b/arrow-data/src/equal/boolean.rs
@@ -16,7 +16,7 @@
 // under the License.
 
 use crate::bit_iterator::BitIndexIterator;
-use crate::data::{contains_nulls, ArrayData};
+use crate::data::{ArrayData, contains_nulls};
 use arrow_buffer::bit_util::get_bit;
 
 use super::utils::{equal_bits, equal_len};
diff --git a/arrow-data/src/equal/dictionary.rs b/arrow-data/src/equal/dictionary.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::{contains_nulls, ArrayData};
+use crate::data::{ArrayData, contains_nulls};
 use arrow_buffer::ArrowNativeType;
 
 use super::equal_range;
diff --git a/arrow-data/src/equal/fixed_list.rs b/arrow-data/src/equal/fixed_list.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::{contains_nulls, ArrayData};
+use crate::data::{ArrayData, contains_nulls};
 use arrow_schema::DataType;
 
 use super::equal_range;
diff --git a/arrow-data/src/equal/list.rs b/arrow-data/src/equal/list.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::{count_nulls, ArrayData};
+use crate::data::{ArrayData, count_nulls};
 use arrow_buffer::ArrowNativeType;
 use num_integer::Integer;
 
diff --git a/arrow-data/src/equal/structure.rs b/arrow-data/src/equal/structure.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::{contains_nulls, ArrayData};
+use crate::data::{ArrayData, contains_nulls};
 
 use super::equal_range;
 
diff --git a/arrow-data/src/equal/utils.rs b/arrow-data/src/equal/utils.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::{contains_nulls, ArrayData};
+use crate::data::{ArrayData, contains_nulls};
 use arrow_buffer::bit_chunk_iterator::BitChunks;
 use arrow_schema::DataType;
 
diff --git a/arrow-data/src/equal/variable_size.rs b/arrow-data/src/equal/variable_size.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::{contains_nulls, ArrayData};
+use crate::data::{ArrayData, contains_nulls};
 use arrow_buffer::ArrowNativeType;
 use num_integer::Integer;
 
diff --git a/arrow-data/src/ffi.rs b/arrow-data/src/ffi.rs
@@ -18,7 +18,7 @@
 //! Contains declarations to bind to the [C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html).
 
 use crate::bit_mask::set_bits;
-use crate::{layout, ArrayData};
+use crate::{ArrayData, layout};
 use arrow_buffer::buffer::NullBuffer;
 use arrow_buffer::{Buffer, MutableBuffer, ScalarBuffer};
 use arrow_schema::DataType;
@@ -71,15 +71,15 @@ unsafe extern "C" fn release_array(array: *mut FFI_ArrowArray) {
     if array.is_null() {
         return;
     }
-    let array = &mut *array;
+    let array = unsafe { &mut *array };
 
     // take ownership of `private_data`, therefore dropping it`
-    let private = Box::from_raw(array.private_data as *mut ArrayPrivateData);
+    let private = unsafe { Box::from_raw(array.private_data as *mut ArrayPrivateData) };
     for child in private.children.iter() {
-        let _ = Box::from_raw(*child);
+        let _ = unsafe { Box::from_raw(*child) };
     }
     if !private.dictionary.is_null() {
-        let _ = Box::from_raw(private.dictionary);
+        let _ = unsafe { Box::from_raw(private.dictionary) };
     }
 
     array.release = None;
@@ -222,7 +222,7 @@ impl FFI_ArrowArray {
     /// [move]: https://arrow.apache.org/docs/format/CDataInterface.html#moving-an-array
     /// [valid]: https://doc.rust-lang.org/std/ptr/index.html#safety
     pub unsafe fn from_raw(array: *mut FFI_ArrowArray) -> Self {
-        std::ptr::replace(array, Self::empty())
+        unsafe { std::ptr::replace(array, Self::empty()) }
     }
 
     /// create an empty `FFI_ArrowArray`, which can be used to import data into
diff --git a/arrow-data/src/transform/boolean.rs b/arrow-data/src/transform/boolean.rs
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::{Extend, _MutableArrayData, utils::resize_for_bits};
-use crate::bit_mask::set_bits;
+use super::{_MutableArrayData, Extend, utils::resize_for_bits};
 use crate::ArrayData;
+use crate::bit_mask::set_bits;
 
 pub(super) fn build_extend(array: &ArrayData) -> Extend<'_> {
     let values = array.buffers()[0].as_slice();
diff --git a/arrow-data/src/transform/fixed_binary.rs b/arrow-data/src/transform/fixed_binary.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::{Extend, _MutableArrayData};
+use super::{_MutableArrayData, Extend};
 use crate::ArrayData;
 use arrow_schema::DataType;
 
diff --git a/arrow-data/src/transform/fixed_size_list.rs b/arrow-data/src/transform/fixed_size_list.rs
@@ -18,7 +18,7 @@
 use crate::ArrayData;
 use arrow_schema::DataType;
 
-use super::{Extend, _MutableArrayData};
+use super::{_MutableArrayData, Extend};
 
 pub(super) fn build_extend(array: &ArrayData) -> Extend<'_> {
     let size = match array.data_type() {
diff --git a/arrow-data/src/transform/list.rs b/arrow-data/src/transform/list.rs
@@ -16,7 +16,7 @@
 // under the License.
 
 use super::{
-    Extend, _MutableArrayData,
+    _MutableArrayData, Extend,
     utils::{extend_offsets, get_last_offset},
 };
 use crate::ArrayData;
diff --git a/arrow-data/src/transform/mod.rs b/arrow-data/src/transform/mod.rs
@@ -20,10 +20,10 @@
 //! Provides utilities for creating, manipulating, and converting Arrow arrays
 //! made of primitive types, strings, and nested types.
 
-use super::{data::new_buffers, ArrayData, ArrayDataBuilder, ByteView};
+use super::{ArrayData, ArrayDataBuilder, ByteView, data::new_buffers};
 use crate::bit_mask::set_bits;
 use arrow_buffer::buffer::{BooleanBuffer, NullBuffer};
-use arrow_buffer::{bit_util, i256, ArrowNativeType, Buffer, MutableBuffer};
+use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, bit_util, i256};
 use arrow_schema::{ArrowError, DataType, IntervalUnit, UnionMode};
 use half::f16;
 use num_integer::Integer;
diff --git a/arrow-data/src/transform/null.rs b/arrow-data/src/transform/null.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::{Extend, _MutableArrayData};
+use super::{_MutableArrayData, Extend};
 use crate::ArrayData;
 
 pub(super) fn build_extend(_: &ArrayData) -> Extend<'_> {
diff --git a/arrow-data/src/transform/primitive.rs b/arrow-data/src/transform/primitive.rs
@@ -20,7 +20,7 @@ use arrow_buffer::ArrowNativeType;
 use std::mem::size_of;
 use std::ops::Add;
 
-use super::{Extend, _MutableArrayData};
+use super::{_MutableArrayData, Extend};
 
 pub(super) fn build_extend<T: ArrowNativeType>(array: &ArrayData) -> Extend<'_> {
     let values = array.buffer::<T>(0);
diff --git a/arrow-data/src/transform/run.rs b/arrow-data/src/transform/run.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::{ArrayData, Extend, _MutableArrayData};
+use super::{_MutableArrayData, ArrayData, Extend};
 use arrow_buffer::{ArrowNativeType, Buffer, ToByteSlice};
 use arrow_schema::DataType;
 use num_traits::CheckedAdd;
diff --git a/arrow-data/src/transform/structure.rs b/arrow-data/src/transform/structure.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::{Extend, _MutableArrayData};
+use super::{_MutableArrayData, Extend};
 use crate::ArrayData;
 
 pub(super) fn build_extend(_: &ArrayData) -> Extend<'_> {
diff --git a/arrow-data/src/transform/union.rs b/arrow-data/src/transform/union.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::{Extend, _MutableArrayData};
+use super::{_MutableArrayData, Extend};
 use crate::ArrayData;
 
 pub(super) fn build_extend_sparse(array: &ArrayData) -> Extend<'_> {
diff --git a/arrow-data/src/transform/utils.rs b/arrow-data/src/transform/utils.rs
diff --git a/arrow-data/src/transform/variable_size.rs b/arrow-data/src/transform/variable_size.rs
