initial

Signed-off-by: Mikhail Kot <to@myrrc.dev>

Author: myrrc

vortex-bench/src/runner.rs

@@ -184,10 +184,11 @@ impl SqlBenchmarkRunner {
         if let Some(expected_counts) = &self.expected_row_counts
             && query_idx < expected_counts.len()
         {
+            let expected = expected_counts[query_idx];
             assert_eq!(
                 row_count,
-                expected_counts[query_idx],
-                "Row count mismatch for query {query_idx} - {engine}:{format}",
+                expected,
+                "Row count mismatch for query {query_idx} - {engine}:{format}, expected {expected}, got {row_count}",
                 engine = self.engine,
             );
         }

vortex-duckdb/cpp/table_function.cpp

@@ -15,6 +15,11 @@ DUCKDB_INCLUDES_BEGIN
 #include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/main/connection.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
+#include "duckdb/planner/expression/bound_operator_expression.hpp"
+#include "duckdb/planner/expression/bound_comparison_expression.hpp"
+#include "duckdb/planner/expression/bound_between_expression.hpp"
+#include "duckdb/planner/expression/bound_conjunction_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
 DUCKDB_INCLUDES_END
 
 using namespace duckdb;
@@ -263,6 +268,19 @@ void function(ClientContext &, TableFunctionInput &input, DataChunk &output) {
     }
 }
 
+/*
+ * Table filter pushdown is used twice in duckdb:
+ *
+ * 1. Planning time: duckdb uses file metadata (filename, hive_partitioning
+ * options in MultiFileReader) to prune files based on filename or hive
+ * partition data i.e. month, year, etc. This happens before any file IO.
+ * We don't use this because we have own file-level pruning in
+ * FileStatsLayoutReader.
+ *
+ * 2. Scan time. As we have filter_pushdown = true, filter expressions are
+ * converted to TableFilterSet and pushed down to Vortex. We convert them to
+ * vortex expressions and use as filter options while initializing the scan.
+ */
 void c_pushdown_complex_filter(ClientContext &,
                                LogicalGet &,
                                FunctionData *bind_data,
@@ -278,8 +296,6 @@ void c_pushdown_complex_filter(ClientContext &,
         if (error_out) {
             throw BinderException(IntoErrString(error_out));
         }
-
-        // If the pushdown complex filter returns true, we can remove the filter from the list.
         iter = pushed ? filters.erase(iter) : std::next(iter);
     }
 }
@@ -381,6 +397,70 @@ InsertionOrderPreservingMap<string> c_to_string(TableFunctionToStringInput &inpu
     return result;
 }
 
+/*
+ * Called either before pushdown_complex_filter or a table filter expression
+ * call. In pushdown_complex_filter we can tell DuckDB we can't push the
+ * filter down by returning Ok(None) but this isn't an option for a table
+ * filter. Be conservative and allow only DuckDB expressions we know will
+ * either always produce a valid Vortex expression or return an error, so no
+ * Ok(None) case.
+ *
+ * See src/convert/expr.rs.
+ */
+bool pushdown_expression(const BaseExpression &expr) {
+    using enum ExpressionClass;
+    switch (expr.GetExpressionClass()) {
+    case BOUND_COLUMN_REF:
+    case BOUND_CONSTANT:
+    case BOUND_REF:
+        return true;
+    case BOUND_COMPARISON: {
+        const auto &comparison = expr.Cast<BoundComparisonExpression>();
+        return pushdown_expression(*comparison.left) && pushdown_expression(*comparison.right);
+    }
+    case BOUND_BETWEEN: {
+        const auto &between = expr.Cast<BoundBetweenExpression>();
+        return pushdown_expression(*between.input) && pushdown_expression(*between.lower) &&
+               pushdown_expression(*between.upper);
+    }
+    case BOUND_CONJUNCTION: {
+        for (const auto &child : expr.Cast<BoundConjunctionExpression>().children) {
+            if (!pushdown_expression(*child)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    case BOUND_FUNCTION: {
+        constexpr std::array<std::string_view, 6> supported =
+            {"struct_extract", "contains", "prefix", "suffix", "~~", "!~~"};
+        const std::string_view name = expr.Cast<BoundFunctionExpression>().function.name;
+        return std::find(supported.begin(), supported.end(), name) != supported.end();
+    }
+    case BOUND_OPERATOR: {
+        switch (expr.GetExpressionType()) {
+        case ExpressionType::OPERATOR_NOT:
+        case ExpressionType::OPERATOR_IS_NULL:
+        case ExpressionType::OPERATOR_IS_NOT_NULL:
+        case ExpressionType::COMPARE_IN:
+        case ExpressionType::COMPARE_NOT_IN:
+            break;
+        default:
+            return false;
+        }
+
+        for (const auto &child : expr.Cast<BoundOperatorExpression>().children) {
+            if (!pushdown_expression(*child)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    default:
+        return false;
+    }
+}
+
 extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const duckdb_vx_tfunc_vtab_t *vtab) {
     D_ASSERT(ffi_db);
     D_ASSERT(vtab);
@@ -395,6 +475,9 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d
     tf.sampling_pushdown = false;
 
     tf.pushdown_complex_filter = c_pushdown_complex_filter;
+    tf.pushdown_expression = [](auto &, const auto &, Expression &expression) {
+        return pushdown_expression(expression);
+    };
     tf.cardinality = c_cardinality;
     tf.get_partition_info = get_partition_info;
     tf.get_partition_data = get_partition_data;