@@ -52,6 +52,7 @@ async fn check_stats_precision_with_filter_pushdown() {
52
52
53
53
let opt = ListingOptions :: new ( Arc :: new ( ParquetFormat :: default ( ) ) ) ;
54
54
let table = get_listing_table ( & table_path, None , & opt) . await ;
55
+
55
56
let ( _, _, state) = get_cache_runtime_state ( ) ;
56
57
let mut options = state. config ( ) . options ( ) . clone ( ) ;
57
58
options. execution . parquet . pushdown_filters = true ;
@@ -60,28 +61,40 @@ async fn check_stats_precision_with_filter_pushdown() {
60
61
let exec = table. scan ( & state, None , & [ ] , None ) . await . unwrap ( ) ;
61
62
assert_eq ! (
62
63
exec. partition_statistics( None ) . unwrap( ) . num_rows,
63
- Precision :: Exact ( 8 )
64
+ Precision :: Exact ( 8 ) ,
65
+ "Stats without filter should be exact"
64
66
) ;
65
67
66
- // Scan with filter pushdown, stats are inexact
67
- // This is a filter that cannot be evaluated by the table provider planning
68
- // (it is not a partition filter) -> will be pushed down to the scan
69
- // with the appropriate optimizer pass.
70
- let filter = Expr :: gt ( col ( "id" ) , lit ( 1 ) ) ;
71
- let exec = table
72
- . scan ( & state, None , & [ filter. clone ( ) ] , None )
68
+ // This is a filter that cannot be evaluated by the table provider scanning
69
+ // (it is not a partition filter). Therefore; it will be pushed down to the
70
+ // source operator after the appropriate optimizer pass.
71
+ let filter_expr = Expr :: gt ( col ( "id" ) , lit ( 1 ) ) ;
72
+ let exec_with_filter = table
73
+ . scan ( & state, None , & [ filter_expr. clone ( ) ] , None )
73
74
. await
74
75
. unwrap ( ) ;
76
+
75
77
let ctx = SessionContext :: new ( ) ;
76
78
let df_schema = DFSchema :: try_from ( table. schema ( ) ) . unwrap ( ) ;
77
- let filter = ctx. create_physical_expr ( filter, & df_schema) . unwrap ( ) ;
78
- let exec =
79
- Arc :: new ( FilterExec :: try_new ( filter, exec) . unwrap ( ) ) as Arc < dyn ExecutionPlan > ;
80
- let exec = FilterPushdown :: new ( ) . optimize ( exec, & options) . unwrap ( ) ;
81
- assert ! ( exec. as_any( ) . is:: <DataSourceExec >( ) ) ; // sanity check that the pushdown did what we expected
79
+ let physical_filter = ctx. create_physical_expr ( filter_expr, & df_schema) . unwrap ( ) ;
80
+
81
+ let filtered_exec =
82
+ Arc :: new ( FilterExec :: try_new ( physical_filter, exec_with_filter) . unwrap ( ) )
83
+ as Arc < dyn ExecutionPlan > ;
84
+
85
+ let optimized_exec = FilterPushdown :: new ( )
86
+ . optimize ( filtered_exec, & options)
87
+ . unwrap ( ) ;
88
+
89
+ assert ! (
90
+ optimized_exec. as_any( ) . is:: <DataSourceExec >( ) ,
91
+ "Sanity check that the pushdown did what we expected"
92
+ ) ;
93
+ // Scan with filter pushdown, stats are inexact
82
94
assert_eq ! (
83
- exec. partition_statistics( None ) . unwrap( ) . num_rows,
84
- Precision :: Inexact ( 8 )
95
+ optimized_exec. partition_statistics( None ) . unwrap( ) . num_rows,
96
+ Precision :: Inexact ( 8 ) ,
97
+ "Stats after filter pushdown should be inexact"
85
98
) ;
86
99
}
87
100
0 commit comments