Skip to content

Commit 3a108b8

Browse files
committed
minor
1 parent 15443d5 commit 3a108b8

File tree

2 files changed

+29
-18
lines changed

2 files changed

+29
-18
lines changed

datafusion/core/tests/parquet/file_statistics.rs

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ async fn check_stats_precision_with_filter_pushdown() {
5252

5353
let opt = ListingOptions::new(Arc::new(ParquetFormat::default()));
5454
let table = get_listing_table(&table_path, None, &opt).await;
55+
5556
let (_, _, state) = get_cache_runtime_state();
5657
let mut options = state.config().options().clone();
5758
options.execution.parquet.pushdown_filters = true;
@@ -60,28 +61,40 @@ async fn check_stats_precision_with_filter_pushdown() {
6061
let exec = table.scan(&state, None, &[], None).await.unwrap();
6162
assert_eq!(
6263
exec.partition_statistics(None).unwrap().num_rows,
63-
Precision::Exact(8)
64+
Precision::Exact(8),
65+
"Stats without filter should be exact"
6466
);
6567

66-
// Scan with filter pushdown, stats are inexact
67-
// This is a filter that cannot be evaluated by the table provider planning
68-
// (it is not a partition filter) -> will be pushed down to the scan
69-
// with the appropriate optimizer pass.
70-
let filter = Expr::gt(col("id"), lit(1));
71-
let exec = table
72-
.scan(&state, None, &[filter.clone()], None)
68+
// This is a filter that cannot be evaluated by the table provider scanning
69+
// (it is not a partition filter). Therefore; it will be pushed down to the
70+
// source operator after the appropriate optimizer pass.
71+
let filter_expr = Expr::gt(col("id"), lit(1));
72+
let exec_with_filter = table
73+
.scan(&state, None, &[filter_expr.clone()], None)
7374
.await
7475
.unwrap();
76+
7577
let ctx = SessionContext::new();
7678
let df_schema = DFSchema::try_from(table.schema()).unwrap();
77-
let filter = ctx.create_physical_expr(filter, &df_schema).unwrap();
78-
let exec =
79-
Arc::new(FilterExec::try_new(filter, exec).unwrap()) as Arc<dyn ExecutionPlan>;
80-
let exec = FilterPushdown::new().optimize(exec, &options).unwrap();
81-
assert!(exec.as_any().is::<DataSourceExec>()); // sanity check that the pushdown did what we expected
79+
let physical_filter = ctx.create_physical_expr(filter_expr, &df_schema).unwrap();
80+
81+
let filtered_exec =
82+
Arc::new(FilterExec::try_new(physical_filter, exec_with_filter).unwrap())
83+
as Arc<dyn ExecutionPlan>;
84+
85+
let optimized_exec = FilterPushdown::new()
86+
.optimize(filtered_exec, &options)
87+
.unwrap();
88+
89+
assert!(
90+
optimized_exec.as_any().is::<DataSourceExec>(),
91+
"Sanity check that the pushdown did what we expected"
92+
);
93+
// Scan with filter pushdown, stats are inexact
8294
assert_eq!(
83-
exec.partition_statistics(None).unwrap().num_rows,
84-
Precision::Inexact(8)
95+
optimized_exec.partition_statistics(None).unwrap().num_rows,
96+
Precision::Inexact(8),
97+
"Stats after filter pushdown should be inexact"
8598
);
8699
}
87100

datafusion/physical-plan/src/filter_pushdown.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,7 @@ impl PredicateSupports {
8484
.into_iter()
8585
.map(|f| match f {
8686
PredicateSupport::Supported(expr) => PredicateSupport::Unsupported(expr),
87-
PredicateSupport::Unsupported(expr) => {
88-
PredicateSupport::Unsupported(expr)
89-
}
87+
u @ PredicateSupport::Unsupported(_) => u,
9088
})
9189
.collect();
9290
Self::new(pushdowns)

0 commit comments

Comments
 (0)