Skip to content

Commit c35ae85

Browse files
committed
Add testing
1 parent 2d66332 commit c35ae85

File tree

1 file changed

+33
-0
lines changed
  • parquet/src/column/writer

1 file changed

+33
-0
lines changed

parquet/src/column/writer/mod.rs

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,6 +1313,10 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> {
13131313
/// Updates column writer metrics with each page metadata.
13141314
#[inline]
13151315
fn update_metrics_for_page(&mut self, page_spec: PageWriteSpec) {
1316+
println!(
1317+
"Update: total_uncompressed_size += {}, total_compressed_size += {}",
1318+
page_spec.uncompressed_size, page_spec.compressed_size
1319+
);
13161320
self.column_metrics.total_uncompressed_size += page_spec.uncompressed_size as u64;
13171321
self.column_metrics.total_compressed_size += page_spec.compressed_size as u64;
13181322
self.column_metrics.total_bytes_written += page_spec.bytes_written;
@@ -4247,4 +4251,33 @@ mod tests {
42474251
.unwrap();
42484252
ColumnDescriptor::new(Arc::new(tpe), max_def_level, max_rep_level, path)
42494253
}
4254+
4255+
#[test]
4256+
fn test_page_v2_snappy_compression_fallback() {
4257+
// Test that PageV2 sets is_compressed to false when Snappy compression increases data size
4258+
let page_writer = TestPageWriter {};
4259+
4260+
// Create WriterProperties with PageV2 and Snappy compression
4261+
let props = WriterProperties::builder()
4262+
.set_writer_version(WriterVersion::PARQUET_2_0)
4263+
// Disable dictionary to ensure data is written directly
4264+
.set_dictionary_enabled(false)
4265+
.set_compression(Compression::SNAPPY)
4266+
.build();
4267+
4268+
let mut column_writer =
4269+
get_test_column_writer::<ByteArrayType>(Box::new(page_writer), 0, 0, Arc::new(props));
4270+
4271+
// Create small, simple data that Snappy compression will likely increase in size
4272+
// due to compression overhead for very small data
4273+
let values = vec![ByteArray::from("a")];
4274+
4275+
column_writer.write_batch(&values, None, None).unwrap();
4276+
4277+
let result = column_writer.close().unwrap();
4278+
assert_eq!(
4279+
result.metadata.uncompressed_size(),
4280+
result.metadata.compressed_size()
4281+
);
4282+
}
42504283
}

0 commit comments

Comments
 (0)