fix for logical schema

Author: etseidl

datafusion/datasource-parquet/src/opener.rs

@@ -180,16 +180,16 @@ impl FileOpener for ParquetOpener {
                 }
             }
 
-            let ordering = reader_metadata
-                .metadata()
-                .file_metadata()
-                .column_orders()
-                .map_or_else(
-                    || vec![ColumnOrdering::Unknown; physical_file_schema.fields().len()],
-                    |column_orders| {
-                        column_orders
-                            .iter()
-                            .map(|order| match order {
+            // Map column ordering from physical schema to logical
+            let ordering: Vec<ColumnOrdering> = if let Some(column_orders) =
+                reader_metadata.metadata().file_metadata().column_orders()
+            {
+                logical_file_schema
+                    .fields()
+                    .iter()
+                    .map(|field| {
+                        match physical_file_schema.index_of(field.name()) {
+                            Ok(idx) => match column_orders[idx] {
                                 ColumnOrder::TYPE_DEFINED_ORDER(sort_order) => {
                                     match sort_order {
                                         SortOrder::SIGNED => ColumnOrdering::Signed,
@@ -202,10 +202,14 @@ impl FileOpener for ParquetOpener {
                                     ColumnOrdering::TotalOrder
                                 }*/
                                 ColumnOrder::UNDEFINED => ColumnOrdering::Unknown,
-                            })
-                            .collect::<Vec<_>>()
-                    },
-                );
+                            },
+                            _ => ColumnOrdering::Unknown,
+                        }
+                    })
+                    .collect::<Vec<_>>()
+            } else {
+                vec![ColumnOrdering::Unknown; logical_file_schema.fields().len()]
+            };
 
             // Build predicates for this specific file
             let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(

datafusion/physical-optimizer/src/pruning.rs

@@ -1511,35 +1511,39 @@ fn build_predicate_expression(
 
     // TODO(ets): this is rather parquet specific...still wonder if this should be done
     //            at the datasource level
-    let colidx = column_index_for_expr(&left, schema)
-        .or_else(|| column_index_for_expr(&right, schema));
-    if let Some(colidx) = colidx {
-        let col_order = column_orderings[colidx];
-
-        // If the ColumnOrder is undefined (as opposed to unknown), we shouldn't be pruning
-        // since min/max are invalid.
-        if col_order == ColumnOrdering::Undefined {
-            dbg!("Cannot prune because column order is undefined");
-            return unhandled_hook.handle(expr);
-        }
 
-        // left and right should have the same type by now, so only check left
-        if left.data_type(schema).is_ok_and(|t| t.is_floating()) {
-            // By the time we've reached this code, we've narrowed down the possible expressions
-            // to binary expressions. Of those allowed by `build_statistics_expr`, we only need
-            // to worry about greater/less than expressions and not equal.
-            match op {
-                Operator::Gt
-                | Operator::GtEq
-                | Operator::Lt
-                | Operator::LtEq
-                | Operator::NotEq => {
-                    if col_order != ColumnOrdering::TotalOrder {
-                        dbg!("Cannot prune floating point column because NaN may be present");
-                        return unhandled_hook.handle(expr);
+    // Sanity check that column orderings match schema
+    if column_orderings.len() == schema.fields().len() {
+        let colidx = column_index_for_expr(&left, schema)
+            .or_else(|| column_index_for_expr(&right, schema));
+        if let Some(colidx) = colidx {
+            let col_order = column_orderings[colidx];
+
+            // If the ColumnOrder is undefined (as opposed to unknown), we shouldn't be pruning
+            // since min/max are invalid.
+            if col_order == ColumnOrdering::Undefined {
+                dbg!("Cannot prune because column order is undefined");
+                return unhandled_hook.handle(expr);
+            }
+
+            // left and right should have the same type by now, so only check left
+            if left.data_type(schema).is_ok_and(|t| t.is_floating()) {
+                // By the time we've reached this code, we've narrowed down the possible expressions
+                // to binary expressions. Of those allowed by `build_statistics_expr`, we only need
+                // to worry about greater/less than expressions and not equal.
+                match op {
+                    Operator::Gt
+                    | Operator::GtEq
+                    | Operator::Lt
+                    | Operator::LtEq
+                    | Operator::NotEq => {
+                        if col_order != ColumnOrdering::TotalOrder {
+                            dbg!("Cannot prune floating point column because NaN may be present");
+                            return unhandled_hook.handle(expr);
+                        }
                     }
+                    _ => (),
                 }
-                _ => (),
             }
         }
     }