[Experimental] Tracking duckdb main branch

.github/workflows/main.yml

@@ -1039,22 +1039,6 @@ jobs:
                       rm -rf ./packages/duckdb-wasm-app/build/
                       yarn workspace @duckdb/duckdb-wasm-app build:release
 
-            - name: Test @duckdb/duckdb-wasm on Chrome
-              shell: bash
-              run: |
-                      CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
-
-            #            - name: Test @duckdb/duckdb-wasm on Firefox
-            #              uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
-            #              with:
-            #                  script: |-
-            #                      yarn workspace @duckdb/duckdb-wasm test:firefox
-
-            - name: Test @duckdb/duckdb-wasm on Node.js
-              shell: bash
-              run: |
-                      yarn workspace @duckdb/duckdb-wasm test:node
-
             #- name: Coverage @duckdb/duckdb-wasm
             #  shell: bash
             #  run: |
@@ -1073,6 +1057,22 @@ jobs:
               run: |
                   ./scripts/npm_measure_lib.sh
 
+            - name: Test @duckdb/duckdb-wasm on Chrome
+              shell: bash
+              run: |
+                      CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
+
+            #            - name: Test @duckdb/duckdb-wasm on Firefox
+            #              uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
+            #              with:
+            #                  script: |-
+            #                      yarn workspace @duckdb/duckdb-wasm test:firefox
+
+            - name: Test @duckdb/duckdb-wasm on Node.js
+              shell: bash
+              run: |
+                      yarn workspace @duckdb/duckdb-wasm test:node
+
             - name: Deploy GitHub Page
               if: github.ref == 'refs/heads/main'
               run: |

extension_config_wasm.cmake

@@ -2,11 +2,11 @@
 # DuckDB-Wasm extension base config
 ################################################################################
 #
+
 duckdb_extension_load(json DONT_LINK)
 duckdb_extension_load(parquet DONT_LINK)
 duckdb_extension_load(autocomplete DONT_LINK)
 
-duckdb_extension_load(fts DONT_LINK)
 duckdb_extension_load(icu DONT_LINK)
 duckdb_extension_load(tpcds DONT_LINK)
 duckdb_extension_load(tpch DONT_LINK)

lib/CMakeLists.txt

@@ -243,10 +243,6 @@ if (DUCKDB_WASM_LOADABLE_EXTENSIONS)
 
   target_link_libraries(duckdb_web duckdb arrow rapidjson ${THREAD_LIBS})
 else()
-  add_library(
-    duckdb_web_fts
-    ${CMAKE_SOURCE_DIR}/src/extensions/fts_extension.cc)
-
   add_library(
     duckdb_web_parquet
     ${CMAKE_SOURCE_DIR}/src/extensions/parquet_extension.cc)
@@ -264,8 +260,7 @@ else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_JSON_EXTENSION")
   endif()
 
-  target_link_libraries(duckdb_web duckdb duckdb_web_fts duckdb_web_parquet ${DUCKDB_WEB_JSON} arrow rapidjson ${THREAD_LIBS})
-  target_link_libraries(duckdb_web_fts duckdb duckdb_fts)
+  target_link_libraries(duckdb_web duckdb duckdb_web_parquet ${DUCKDB_WEB_JSON} arrow rapidjson ${THREAD_LIBS})
   target_link_libraries(duckdb_web_parquet duckdb duckdb_parquet)
   target_link_libraries(duckdb_web_json duckdb duckdb_json)
 endif()
@@ -275,6 +270,8 @@ endif()
 
 # We need "-s WARN_ON_UNDEFINED_SYMBOLS=0" to instantiate the module with our
 # own imports.
+     # --pre-js "/Users/carlo/databricks/duckdb_delta/file_system_packager.js" \
+#
 if(EMSCRIPTEN)
   add_executable(duckdb_wasm ${CMAKE_SOURCE_DIR}/src/wasm_main.cc)
   target_link_libraries(duckdb_wasm duckdb_web ${THREAD_LIBS})
@@ -371,7 +368,7 @@ if(NOT EMSCRIPTEN)
       ${CMAKE_SOURCE_DIR}/test/webdb_test.cc
       ${CMAKE_SOURCE_DIR}/test/tester.cc)
 
-  set(TEST_LIBS duckdb_web duckdb_web_fts duckdb_web_parquet ${DUCKDB_WEB_JSON} gtest gmock gflags ${THREAD_LIBS})
+  set(TEST_LIBS duckdb_web duckdb_web_parquet ${DUCKDB_WEB_JSON} gtest gmock gflags ${THREAD_LIBS})
 
   add_executable(tester ${TEST_CC})
   target_link_libraries(tester ${TEST_LIBS})

lib/cmake/duckdb.cmake

@@ -15,7 +15,7 @@ endif()
 set(DUCKDB_CXX_FLAGS "${DUCKDB_CXX_FLAGS} -Wno-unqualified-std-cast-call -DDUCKDB_DEBUG_NO_SAFETY -DDUCKDB_FROM_DUCKDB_WASM")
 message("DUCKDB_CXX_FLAGS=${DUCKDB_CXX_FLAGS}")
 
-set(DUCKDB_EXTENSIONS "fts;json")
+set(DUCKDB_EXTENSIONS "json;core_functions")
 # Escape semicolons in DUCKDB_EXTENSIONS before passing to ExternalProject_Add
 string(REPLACE ";" "$<SEMICOLON>" DUCKDB_EXTENSIONS_PACKED "${DUCKDB_EXTENSIONS}")
 
@@ -46,8 +46,10 @@ ExternalProject_Add(
              -DDISABLE_BUILTIN_EXTENSIONS=TRUE
              -DUSE_WASM_THREADS=${USE_WASM_THREADS}
              -DDUCKDB_EXPLICIT_PLATFORM=${DUCKDB_EXPLICIT_PLATFORM}
+             -DSMALLER_BINARY=1
   BUILD_BYPRODUCTS
     <INSTALL_DIR>/lib/libduckdb_re2.a
+    <INSTALL_DIR>/lib/libduckdb_zstd.a
     <INSTALL_DIR>/lib/libduckdb_static.a
     <INSTALL_DIR>/lib/libduckdb_fmt.a
     <INSTALL_DIR>/lib/libduckdb_fsst.a
@@ -59,7 +61,7 @@ ExternalProject_Add(
     <INSTALL_DIR>/lib/libduckdb_utf8proc.a
     <INSTALL_DIR>/lib/libduckdb_fastpforlib.a
     <INSTALL_DIR>/lib/libparquet_extension.a
-    <INSTALL_DIR>/lib/libfts_extension.a
+    <INSTALL_DIR>/lib/libcore_functions_extension.a
     <INSTALL_DIR>/lib/libjson_extension.a)
 
 ExternalProject_Get_Property(duckdb_ep install_dir)
@@ -81,6 +83,7 @@ set_property(TARGET duckdb PROPERTY IMPORTED_LOCATION ${DUCKDB_LIBRARY_PATH})
 target_link_libraries(
   duckdb
   INTERFACE ${install_dir}/lib/libduckdb_re2.a
+  INTERFACE ${install_dir}/lib/libduckdb_zstd.a
   INTERFACE ${install_dir}/lib/libduckdb_fmt.a
   INTERFACE ${install_dir}/lib/libduckdb_fsst.a
   INTERFACE ${install_dir}/lib/libduckdb_hyperloglog.a
@@ -90,6 +93,7 @@ target_link_libraries(
   INTERFACE ${install_dir}/lib/libduckdb_pg_query.a
   INTERFACE ${install_dir}/lib/libduckdb_utf8proc.a
   INTERFACE ${install_dir}/lib/libduckdb_fastpforlib.a
+  INTERFACE ${install_dir}/lib/libcore_functions_extension.a
   INTERFACE dl)
 
 target_include_directories(
@@ -105,10 +109,6 @@ target_include_directories(
   INTERFACE ${DUCKDB_SOURCE_DIR}/third_party/thrift
   INTERFACE ${DUCKDB_SOURCE_DIR}/third_party/zstd)
 
-add_library(duckdb_fts STATIC IMPORTED)
-set_property(TARGET duckdb_fts PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libfts_extension.a)
-target_include_directories(duckdb_fts INTERFACE ${DUCKDB_SOURCE_DIR}/extension/fts/include)
-
 add_library(duckdb_parquet STATIC IMPORTED)
 set_property(TARGET duckdb_parquet PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libparquet_extension.a)
 target_include_directories(duckdb_parquet INTERFACE ${DUCKDB_SOURCE_DIR}/extension/parquet/include)
@@ -117,7 +117,10 @@ add_library(duckdb_json STATIC IMPORTED)
 set_property(TARGET duckdb_json PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libjson_extension.a)
 target_include_directories(duckdb_json INTERFACE ${DUCKDB_SOURCE_DIR}/extension/json/include)
 
+add_library(duckdb_core_functions STATIC IMPORTED)
+set_property(TARGET duckdb_core_functions PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libcore_functions_extension.a)
+target_include_directories(duckdb_core_functions INTERFACE ${DUCKDB_SOURCE_DIR}/extension/json/include)
+
 add_dependencies(duckdb duckdb_ep)
-add_dependencies(duckdb_fts duckdb_ep)
 add_dependencies(duckdb_parquet duckdb_ep)
 add_dependencies(duckdb_json duckdb_ep)

lib/include/duckdb/web/io/web_filesystem.h

@@ -132,7 +132,7 @@ class WebFileSystem : public duckdb::FileSystem {
        public:
         /// Constructor
         WebFileHandle(std::shared_ptr<WebFile> file)
-            : duckdb::FileHandle(file->GetFileSystem(), file->GetFileName()),
+            : duckdb::FileHandle(file->GetFileSystem(), file->GetFileName(), FileOpenFlags::FILE_FLAGS_READ),
               file_(file),
               readahead_(nullptr),
               position_(0) {

lib/src/io/buffered_filesystem.cc

@@ -42,7 +42,7 @@ void BufferedFileHandle::Close() { file_ref_->Release(); }
 /// Constructor
 BufferedFileHandle::BufferedFileHandle(duckdb::FileSystem &file_system,
                                        std::shared_ptr<FilePageBuffer::FileRef> file_buffers)
-    : duckdb::FileHandle(file_system, std::string{file_buffers->GetPath()}),
+    : duckdb::FileHandle(file_system, std::string{file_buffers->GetPath()}, FileOpenFlags::FILE_FLAGS_READ),
       file_ref_(std::move(file_buffers)),
       file_position_(0) {}
 

lib/src/io/memory_filesystem.cc

@@ -17,7 +17,8 @@ MemoryFileSystem::FileBuffer::FileBuffer(size_t id, std::string path, std::vecto
     : file_id(id), file_path(std::move(path)), buffer(std::move(buffer)), handles() {}
 /// Constructor
 MemoryFileSystem::FileHandle::FileHandle(MemoryFileSystem &file_system, FileBuffer &buffer, size_t position)
-    : duckdb::FileHandle(file_system, buffer.file_path),
+    : duckdb::FileHandle(file_system, buffer.file_path,
+                         FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE),
       file_system_(file_system),
       buffer_(buffer),
       position_(position) {}

lib/src/webdb.cc

@@ -45,7 +45,6 @@
 #include "duckdb/web/config.h"
 #include "duckdb/web/csv_insert_options.h"
 #include "duckdb/web/environment.h"
-#include "duckdb/web/extensions/fts_extension.h"
 #include "duckdb/web/extensions/json_extension.h"
 #include "duckdb/web/extensions/parquet_extension.h"
 #include "duckdb/web/functions/table_function_relation.h"
@@ -105,6 +104,7 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::MaterializeQuer
     // Configure the output writer
     ArrowSchema raw_schema;
     ClientProperties options;
+    unordered_map<idx_t, const shared_ptr<ArrowTypeExtensionData>> extension_type_cast;
     options.arrow_offset_size = ArrowOffsetSize::REGULAR;
     ArrowConverter::ToArrowSchema(&raw_schema, result->types, result->names, options);
     ARROW_ASSIGN_OR_RAISE(auto schema, arrow::ImportSchema(&raw_schema));
@@ -119,7 +119,7 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::MaterializeQuer
     for (auto chunk = result->Fetch(); !!chunk && chunk->size() > 0; chunk = result->Fetch()) {
         // Import the data chunk as record batch
         ArrowArray array;
-        ArrowConverter::ToArrowArray(*chunk, &array, options);
+        ArrowConverter::ToArrowArray(*chunk, &array, options, extension_type_cast);
         // Import the record batch
         ARROW_ASSIGN_OR_RAISE(auto batch, arrow::ImportRecordBatch(&array, schema));
         // Patch the record batch
@@ -252,8 +252,9 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::FetchQueryResul
         // Serialize the record batch
         ArrowArray array;
         ClientProperties arrow_options;
+        unordered_map<idx_t, const shared_ptr<ArrowTypeExtensionData>> extension_type_cast;
         arrow_options.arrow_offset_size = ArrowOffsetSize::REGULAR;
-        ArrowConverter::ToArrowArray(*chunk, &array, arrow_options);
+        ArrowConverter::ToArrowArray(*chunk, &array, arrow_options, extension_type_cast);
         ARROW_ASSIGN_OR_RAISE(auto batch, arrow::ImportRecordBatch(&array, current_schema_));
         // Patch the record batch
         ARROW_ASSIGN_OR_RAISE(batch, patchRecordBatch(batch, current_schema_patched_, webdb_.config_->query));
@@ -831,7 +832,6 @@ arrow::Status WebDB::Open(std::string_view args_json) {
         auto db = make_shared_ptr<duckdb::DuckDB>(config_->path, &db_config);
 #ifndef WASM_LOADABLE_EXTENSIONS
         duckdb_web_parquet_init(db.get());
-        duckdb_web_fts_init(db.get());
 #if defined(DUCKDB_JSON_EXTENSION)
         duckdb_web_json_init(db.get());
 #endif

patches/duckdb/add_constructor_client_properties.patch

@@ -0,0 +1,12 @@
+diff --git a/src/include/duckdb/main/client_properties.hpp b/src/include/duckdb/main/client_properties.hpp
+index a4038a7353..d3a269ea36 100644
+--- a/src/include/duckdb/main/client_properties.hpp
++++ b/src/include/duckdb/main/client_properties.hpp
+@@ -26,6 +26,7 @@ struct ClientProperties {
+ 	      arrow_use_list_view(arrow_use_list_view_p), produce_arrow_string_view(produce_arrow_string_view_p),
+ 	      arrow_lossless_conversion(lossless_conversion), client_context(client_context) {
+ 	}
++	ClientProperties() {}
+
+ 	string time_zone = "UTC";
+ 	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;

patches/duckdb/duckdb_smaller_binary_no_select.patch

@@ -0,0 +1,37 @@
+diff --git a/src/execution/expression_executor.cpp b/src/execution/expression_executor.cpp
+index 63a24f479e..eadf1dea0d 100644
+--- a/src/execution/expression_executor.cpp
++++ b/src/execution/expression_executor.cpp
+@@ -236,7 +236,7 @@ idx_t ExpressionExecutor::Select(const Expression &expr, ExpressionState *state,
+ 	D_ASSERT(true_sel || false_sel);
+ 	D_ASSERT(expr.return_type.id() == LogicalTypeId::BOOLEAN);
+	switch (expr.GetExpressionClass()) {
+-#ifndef DUCKDB_SMALLER_BINARY
++#ifndef DUCKDB_SMALLER_BINARY_NO_SELECT
+ 	case ExpressionClass::BOUND_BETWEEN:
+ 		return Select(expr.Cast<BoundBetweenExpression>(), state, sel, count, true_sel, false_sel);
+ #endif
+diff --git a/src/execution/expression_executor/execute_between.cpp b/src/execution/expression_executor/execute_between.cpp
+index 3418351362..52cf454530 100644
+--- a/src/execution/expression_executor/execute_between.cpp
++++ b/src/execution/expression_executor/execute_between.cpp
+@@ -7,7 +7,7 @@
+
+ namespace duckdb {
+
+-#ifndef DUCKDB_SMALLER_BINARY
++#ifndef DUCKDB_SMALLER_BINARY_NO_SELECT
+ struct BothInclusiveBetweenOperator {
+ 	template <class T>
+ 	static inline bool Operation(T input, T lower, T upper) {
+@@ -133,8 +133,8 @@ void ExpressionExecutor::Execute(const BoundBetweenExpression &expr, ExpressionS
+
+ idx_t ExpressionExecutor::Select(const BoundBetweenExpression &expr, ExpressionState *state, const SelectionVector *sel,
+                                  idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) {
+-#ifdef DUCKDB_SMALLER_BINARY
+-	throw InternalException("ExpressionExecutor::Select not available with DUCKDB_SMALLER_BINARY");
++#ifdef DUCKDB_SMALLER_BINARY_NO_SELECT
++	throw InternalException("ExpressionExecutor::Select not available with DUCKDB_SMALLER_BINARY_NO_SELECT");
+ #else
+ 	// resolve the children
+ 	Vector input(state->intermediate_chunk.data[0]);

patches/duckdb/extension_install_rework.patch

@@ -5,7 +5,7 @@ index 2a6fffa994..f4ef687ddd 100644
 @@ -96,6 +96,10 @@ private:
  	shared_ptr<DatabaseCacheEntry> db_cache_entry;
 
- 	duckdb_ext_api_v0 (*create_api_v0)();
+ 	duckdb_ext_api_v1 (*create_api_v1)();
 +public:
 +	static void SetPreferredRepository(const string& extension, const string &repository);
 +	static string GetPreferredRepository(const string& extension);

patches/duckdb/fmt_fmt.patch

@@ -0,0 +1,49 @@
+commit 0639b847a4d7361447b4e8b729dee2b6b2685f8c
+Author: Carlo Piovesan <[email protected]>
+Date:   Sat Dec 14 03:53:32 2024 +0100
+
+    Adapt fmt to -Wdeprecated-literal-operator
+
+    That's turned on in DuckDB-Wasm, and adds unnecessary noise
+
+diff --git a/third_party/fmt/include/fmt/format.h b/third_party/fmt/include/fmt/format.h
+index ad4fe83ab4..e92437a2ed 100644
+--- a/third_party/fmt/include/fmt/format.h
++++ b/third_party/fmt/include/fmt/format.h
+@@ -548,7 +548,7 @@ class u8string_view : public basic_string_view<fmt_char8_t> {
+
+ #if FMT_USE_USER_DEFINED_LITERALS
+ inline namespace literals {
+-inline u8string_view operator"" _u(const char* s, std::size_t n) {
++inline u8string_view operator""_u(const char* s, std::size_t n) {
+   return {s, n};
+ }
+ }  // namespace literals
+@@ -3342,11 +3342,11 @@ FMT_CONSTEXPR internal::udl_formatter<Char, CHARS...> operator""_format() {
+     std::string message = "The answer is {}"_format(42);
+   \endrst
+  */
+-FMT_CONSTEXPR internal::udl_formatter<char> operator"" _format(const char* s,
++FMT_CONSTEXPR internal::udl_formatter<char> operator""_format(const char* s,
+                                                                std::size_t n) {
+   return {{s, n}};
+ }
+-FMT_CONSTEXPR internal::udl_formatter<wchar_t> operator"" _format(
++FMT_CONSTEXPR internal::udl_formatter<wchar_t> operator""_format(
+     const wchar_t* s, std::size_t n) {
+   return {{s, n}};
+ }
+@@ -3362,11 +3362,11 @@ FMT_CONSTEXPR internal::udl_formatter<wchar_t> operator"" _format(
+     fmt::print("Elapsed time: {s:.2f} seconds", "s"_a=1.23);
+   \endrst
+  */
+-FMT_CONSTEXPR internal::udl_arg<char> operator"" _a(const char* s,
++FMT_CONSTEXPR internal::udl_arg<char> operator""_a(const char* s,
+                                                     std::size_t n) {
+   return {{s, n}};
+ }
+-FMT_CONSTEXPR internal::udl_arg<wchar_t> operator"" _a(const wchar_t* s,
++FMT_CONSTEXPR internal::udl_arg<wchar_t> operator""_a(const wchar_t* s,
+                                                        std::size_t n) {
+   return {{s, n}};
+ }

patches/duckdb/hardcode_abi_type.patch

@@ -0,0 +1,13 @@
+diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp
+index a7a7c62fef..d560a3b3b3 100644
+--- a/src/main/extension/extension_load.cpp
++++ b/src/main/extension/extension_load.cpp
+@@ -451,7 +451,7 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
+ 	result.filebase = lowercase_extension_name;
+ 	result.filename = filename;
+ 	result.lib_hdl = lib_hdl;
+-	result.abi_type = parsed_metadata.abi_type;
++	result.abi_type = ExtensionABIType::CPP;
+
+ 	if (!direct_load) {
+ 		auto info_file_name = filename + ".info";