Skip to content

fix(parquet_writer): correctly update upper bound #1520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 29, 2025
Merged

fix(parquet_writer): correctly update upper bound #1520

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions crates/iceberg/src/writer/file_writer/parquet_writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ impl MinMaxColAggregator {
self.upper_bounds
.entry(field_id)
.and_modify(|e| {
if *e > datum {
if *e < datum {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice catch! can we also add an unit test for this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will look into it, now it's tested as part of test_parquet_writer

Copy link
Contributor Author

@Erigara Erigara Jul 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@CTTY sorry for the delay i've added test_min_max_aggregator test

*e = datum.clone()
}
})
Expand Down Expand Up @@ -664,6 +664,7 @@ mod tests {
use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef};
use arrow_select::concat::concat_batches;
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
use parquet::file::statistics::ValueStatistics;
use rust_decimal::Decimal;
use tempfile::TempDir;
use uuid::Uuid;
Expand Down Expand Up @@ -853,7 +854,9 @@ mod tests {

// write data
let mut pw = ParquetWriterBuilder::new(
WriterProperties::builder().build(),
WriterProperties::builder()
.set_max_row_group_size(128)
.build(),
Arc::new(to_write.schema().as_ref().try_into().unwrap()),
file_io.clone(),
location_gen,
Expand Down Expand Up @@ -2284,4 +2287,39 @@ mod tests {
// Check that file should have been deleted.
assert_eq!(std::fs::read_dir(temp_dir.path()).unwrap().count(), 0);
}

#[test]
fn test_min_max_aggregator() {
let schema = Arc::new(
Schema::builder()
.with_schema_id(1)
.with_fields(vec![
NestedField::required(0, "col", Type::Primitive(PrimitiveType::Int))
.with_id(0)
.into(),
])
.build()
.expect("Failed to create schema"),
);
let mut min_max_agg = MinMaxColAggregator::new(schema);
let create_statistics =
|min, max| Statistics::Int32(ValueStatistics::new(min, max, None, None, false));
min_max_agg
.update(0, create_statistics(None, Some(42)))
.unwrap();
min_max_agg
.update(0, create_statistics(Some(0), Some(i32::MAX)))
.unwrap();
min_max_agg
.update(0, create_statistics(Some(i32::MIN), None))
.unwrap();
min_max_agg
.update(0, create_statistics(None, None))
.unwrap();

let (lower_bounds, upper_bounds) = min_max_agg.produce();

assert_eq!(lower_bounds, HashMap::from([(0, Datum::int(i32::MIN))]));
assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))]));
}
}
Loading