Merge commit '37ac77d1896b11246b127c5384c640e9f56454e5' into HEAD

.github/workflows/main.yml

@@ -1039,22 +1039,6 @@ jobs:
                       rm -rf ./packages/duckdb-wasm-app/build/
                       yarn workspace @duckdb/duckdb-wasm-app build:release
 
-            - name: Test @duckdb/duckdb-wasm on Chrome
-              shell: bash
-              run: |
-                      CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
-            
-            #            - name: Test @duckdb/duckdb-wasm on Firefox
-            #              uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
-            #              with:
-            #                  script: |-
-            #                      yarn workspace @duckdb/duckdb-wasm test:firefox
-
-            - name: Test @duckdb/duckdb-wasm on Node.js
-              shell: bash
-              run: |
-                      yarn workspace @duckdb/duckdb-wasm test:node
-
             #- name: Coverage @duckdb/duckdb-wasm
             #  shell: bash
             #  run: |
@@ -1073,6 +1057,22 @@ jobs:
               run: |
                   ./scripts/npm_measure_lib.sh
 
+            - name: Test @duckdb/duckdb-wasm on Chrome
+              shell: bash
+              run: |
+                      CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
+            
+            #            - name: Test @duckdb/duckdb-wasm on Firefox
+            #              uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
+            #              with:
+            #                  script: |-
+            #                      yarn workspace @duckdb/duckdb-wasm test:firefox
+
+            - name: Test @duckdb/duckdb-wasm on Node.js
+              shell: bash
+              run: |
+                      yarn workspace @duckdb/duckdb-wasm test:node
+
             - name: Deploy GitHub Page
               if: github.ref == 'refs/heads/main'
               run: |

README.md

@@ -36,7 +36,7 @@ Try it out at [shell.duckdb.org](https://shell.duckdb.org) or [external third pa
 
 ## DuckDB and DuckDB-Wasm
 
-DuckDB-Wasm is currently based on DuckDB v1.1.1.
+DuckDB-Wasm is currently based on DuckDB v1.2.0.
 
 Relevant differences:
 * HTTP stack is different between native and Wasm versions of DuckDB. Most relevant are:

extension_config_wasm.cmake

@@ -2,11 +2,11 @@
 # DuckDB-Wasm extension base config
 ################################################################################
 #
+
 duckdb_extension_load(json DONT_LINK)
 duckdb_extension_load(parquet DONT_LINK)
 duckdb_extension_load(autocomplete DONT_LINK)
 
-duckdb_extension_load(fts DONT_LINK)
 duckdb_extension_load(icu DONT_LINK)
 duckdb_extension_load(tpcds DONT_LINK)
 duckdb_extension_load(tpch DONT_LINK)

lib/CMakeLists.txt

@@ -243,10 +243,6 @@ if (DUCKDB_WASM_LOADABLE_EXTENSIONS)
 
   target_link_libraries(duckdb_web duckdb arrow rapidjson ${THREAD_LIBS})
 else()
-  add_library(
-    duckdb_web_fts
-    ${CMAKE_SOURCE_DIR}/src/extensions/fts_extension.cc)
-
   add_library(
     duckdb_web_parquet
     ${CMAKE_SOURCE_DIR}/src/extensions/parquet_extension.cc)
@@ -264,8 +260,7 @@ else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_JSON_EXTENSION")
   endif()
 
-  target_link_libraries(duckdb_web duckdb duckdb_web_fts duckdb_web_parquet ${DUCKDB_WEB_JSON} arrow rapidjson ${THREAD_LIBS})
-  target_link_libraries(duckdb_web_fts duckdb duckdb_fts)
+  target_link_libraries(duckdb_web duckdb duckdb_web_parquet ${DUCKDB_WEB_JSON} arrow rapidjson ${THREAD_LIBS})
   target_link_libraries(duckdb_web_parquet duckdb duckdb_parquet)
   target_link_libraries(duckdb_web_json duckdb duckdb_json)
 endif()
@@ -275,6 +270,8 @@ endif()
 
 # We need "-s WARN_ON_UNDEFINED_SYMBOLS=0" to instantiate the module with our
 # own imports.
+     # --pre-js "/Users/carlo/databricks/duckdb_delta/file_system_packager.js" \
+#
 if(EMSCRIPTEN)
   add_executable(duckdb_wasm ${CMAKE_SOURCE_DIR}/src/wasm_main.cc)
   target_link_libraries(duckdb_wasm duckdb_web ${THREAD_LIBS})
@@ -371,7 +368,7 @@ if(NOT EMSCRIPTEN)
       ${CMAKE_SOURCE_DIR}/test/webdb_test.cc
       ${CMAKE_SOURCE_DIR}/test/tester.cc)
 
-  set(TEST_LIBS duckdb_web duckdb_web_fts duckdb_web_parquet ${DUCKDB_WEB_JSON} gtest gmock gflags ${THREAD_LIBS})
+  set(TEST_LIBS duckdb_web duckdb_web_parquet ${DUCKDB_WEB_JSON} gtest gmock gflags ${THREAD_LIBS})
 
   add_executable(tester ${TEST_CC})
   target_link_libraries(tester ${TEST_LIBS})

lib/cmake/duckdb.cmake

@@ -15,7 +15,7 @@ endif()
 set(DUCKDB_CXX_FLAGS "${DUCKDB_CXX_FLAGS} -Wno-unqualified-std-cast-call -DDUCKDB_DEBUG_NO_SAFETY -DDUCKDB_FROM_DUCKDB_WASM")
 message("DUCKDB_CXX_FLAGS=${DUCKDB_CXX_FLAGS}")
 
-set(DUCKDB_EXTENSIONS "fts;json")
+set(DUCKDB_EXTENSIONS "json;core_functions")
 # Escape semicolons in DUCKDB_EXTENSIONS before passing to ExternalProject_Add
 string(REPLACE ";" "$<SEMICOLON>" DUCKDB_EXTENSIONS_PACKED "${DUCKDB_EXTENSIONS}")
 
@@ -46,8 +46,10 @@ ExternalProject_Add(
              -DDISABLE_BUILTIN_EXTENSIONS=TRUE
              -DUSE_WASM_THREADS=${USE_WASM_THREADS}
              -DDUCKDB_EXPLICIT_PLATFORM=${DUCKDB_EXPLICIT_PLATFORM}
+             -DSMALLER_BINARY=1
   BUILD_BYPRODUCTS
     <INSTALL_DIR>/lib/libduckdb_re2.a
+    <INSTALL_DIR>/lib/libduckdb_zstd.a
     <INSTALL_DIR>/lib/libduckdb_static.a
     <INSTALL_DIR>/lib/libduckdb_fmt.a
     <INSTALL_DIR>/lib/libduckdb_fsst.a
@@ -59,7 +61,7 @@ ExternalProject_Add(
     <INSTALL_DIR>/lib/libduckdb_utf8proc.a
     <INSTALL_DIR>/lib/libduckdb_fastpforlib.a
     <INSTALL_DIR>/lib/libparquet_extension.a
-    <INSTALL_DIR>/lib/libfts_extension.a
+    <INSTALL_DIR>/lib/libcore_functions_extension.a
     <INSTALL_DIR>/lib/libjson_extension.a)
 
 ExternalProject_Get_Property(duckdb_ep install_dir)
@@ -81,6 +83,7 @@ set_property(TARGET duckdb PROPERTY IMPORTED_LOCATION ${DUCKDB_LIBRARY_PATH})
 target_link_libraries(
   duckdb
   INTERFACE ${install_dir}/lib/libduckdb_re2.a
+  INTERFACE ${install_dir}/lib/libduckdb_zstd.a
   INTERFACE ${install_dir}/lib/libduckdb_fmt.a
   INTERFACE ${install_dir}/lib/libduckdb_fsst.a
   INTERFACE ${install_dir}/lib/libduckdb_hyperloglog.a
@@ -90,6 +93,7 @@ target_link_libraries(
   INTERFACE ${install_dir}/lib/libduckdb_pg_query.a
   INTERFACE ${install_dir}/lib/libduckdb_utf8proc.a
   INTERFACE ${install_dir}/lib/libduckdb_fastpforlib.a
+  INTERFACE ${install_dir}/lib/libcore_functions_extension.a
   INTERFACE dl)
 
 target_include_directories(
@@ -105,10 +109,6 @@ target_include_directories(
   INTERFACE ${DUCKDB_SOURCE_DIR}/third_party/thrift
   INTERFACE ${DUCKDB_SOURCE_DIR}/third_party/zstd)
 
-add_library(duckdb_fts STATIC IMPORTED)
-set_property(TARGET duckdb_fts PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libfts_extension.a)
-target_include_directories(duckdb_fts INTERFACE ${DUCKDB_SOURCE_DIR}/extension/fts/include)
-
 add_library(duckdb_parquet STATIC IMPORTED)
 set_property(TARGET duckdb_parquet PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libparquet_extension.a)
 target_include_directories(duckdb_parquet INTERFACE ${DUCKDB_SOURCE_DIR}/extension/parquet/include)
@@ -117,7 +117,10 @@ add_library(duckdb_json STATIC IMPORTED)
 set_property(TARGET duckdb_json PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libjson_extension.a)
 target_include_directories(duckdb_json INTERFACE ${DUCKDB_SOURCE_DIR}/extension/json/include)
 
+add_library(duckdb_core_functions STATIC IMPORTED)
+set_property(TARGET duckdb_core_functions PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libcore_functions_extension.a)
+target_include_directories(duckdb_core_functions INTERFACE ${DUCKDB_SOURCE_DIR}/extension/json/include)
+
 add_dependencies(duckdb duckdb_ep)
-add_dependencies(duckdb_fts duckdb_ep)
 add_dependencies(duckdb_parquet duckdb_ep)
 add_dependencies(duckdb_json duckdb_ep)

lib/include/duckdb/web/io/web_filesystem.h

@@ -132,7 +132,7 @@ class WebFileSystem : public duckdb::FileSystem {
        public:
         /// Constructor
         WebFileHandle(std::shared_ptr<WebFile> file)
-            : duckdb::FileHandle(file->GetFileSystem(), file->GetFileName()),
+            : duckdb::FileHandle(file->GetFileSystem(), file->GetFileName(), FileOpenFlags::FILE_FLAGS_READ),
               file_(file),
               readahead_(nullptr),
               position_(0) {

lib/src/io/buffered_filesystem.cc

@@ -42,7 +42,7 @@ void BufferedFileHandle::Close() { file_ref_->Release(); }
 /// Constructor
 BufferedFileHandle::BufferedFileHandle(duckdb::FileSystem &file_system,
                                        std::shared_ptr<FilePageBuffer::FileRef> file_buffers)
-    : duckdb::FileHandle(file_system, std::string{file_buffers->GetPath()}),
+    : duckdb::FileHandle(file_system, std::string{file_buffers->GetPath()}, FileOpenFlags::FILE_FLAGS_READ),
       file_ref_(std::move(file_buffers)),
       file_position_(0) {}
 

lib/src/io/memory_filesystem.cc

@@ -17,7 +17,8 @@ MemoryFileSystem::FileBuffer::FileBuffer(size_t id, std::string path, std::vecto
     : file_id(id), file_path(std::move(path)), buffer(std::move(buffer)), handles() {}
 /// Constructor
 MemoryFileSystem::FileHandle::FileHandle(MemoryFileSystem &file_system, FileBuffer &buffer, size_t position)
-    : duckdb::FileHandle(file_system, buffer.file_path),
+    : duckdb::FileHandle(file_system, buffer.file_path,
+                         FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE),
       file_system_(file_system),
       buffer_(buffer),
       position_(position) {}

lib/src/webdb.cc

@@ -45,7 +45,6 @@
 #include "duckdb/web/config.h"
 #include "duckdb/web/csv_insert_options.h"
 #include "duckdb/web/environment.h"
-#include "duckdb/web/extensions/fts_extension.h"
 #include "duckdb/web/extensions/json_extension.h"
 #include "duckdb/web/extensions/parquet_extension.h"
 #include "duckdb/web/functions/table_function_relation.h"
@@ -104,7 +103,8 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::MaterializeQuer
 
     // Configure the output writer
     ArrowSchema raw_schema;
-    ClientProperties options;
+    ClientProperties options("UTC", ArrowOffsetSize::REGULAR, false, false, false, connection_.context);
+    unordered_map<idx_t, const shared_ptr<ArrowTypeExtensionData>> extension_type_cast;
     options.arrow_offset_size = ArrowOffsetSize::REGULAR;
     ArrowConverter::ToArrowSchema(&raw_schema, result->types, result->names, options);
     ARROW_ASSIGN_OR_RAISE(auto schema, arrow::ImportSchema(&raw_schema));
@@ -119,7 +119,7 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::MaterializeQuer
     for (auto chunk = result->Fetch(); !!chunk && chunk->size() > 0; chunk = result->Fetch()) {
         // Import the data chunk as record batch
         ArrowArray array;
-        ArrowConverter::ToArrowArray(*chunk, &array, options);
+        ArrowConverter::ToArrowArray(*chunk, &array, options, extension_type_cast);
         // Import the record batch
         ARROW_ASSIGN_OR_RAISE(auto batch, arrow::ImportRecordBatch(&array, schema));
         // Patch the record batch
@@ -139,7 +139,7 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::StreamQueryResu
 
     // Import the schema
     ArrowSchema raw_schema;
-    ClientProperties options;
+    ClientProperties options("UTC", ArrowOffsetSize::REGULAR, false, false, false, connection_.context);
     options.arrow_offset_size = ArrowOffsetSize::REGULAR;
     ArrowConverter::ToArrowSchema(&raw_schema, current_query_result_->types, current_query_result_->names, options);
     ARROW_ASSIGN_OR_RAISE(current_schema_, arrow::ImportSchema(&raw_schema));
@@ -251,9 +251,10 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::FetchQueryResul
 
         // Serialize the record batch
         ArrowArray array;
-        ClientProperties arrow_options;
+        ClientProperties arrow_options("UTC", ArrowOffsetSize::REGULAR, false, false, false, connection_.context);
+        unordered_map<idx_t, const shared_ptr<ArrowTypeExtensionData>> extension_type_cast;
         arrow_options.arrow_offset_size = ArrowOffsetSize::REGULAR;
-        ArrowConverter::ToArrowArray(*chunk, &array, arrow_options);
+        ArrowConverter::ToArrowArray(*chunk, &array, arrow_options, extension_type_cast);
         ARROW_ASSIGN_OR_RAISE(auto batch, arrow::ImportRecordBatch(&array, current_schema_));
         // Patch the record batch
         ARROW_ASSIGN_OR_RAISE(batch, patchRecordBatch(batch, current_schema_patched_, webdb_.config_->query));
@@ -832,7 +833,6 @@ arrow::Status WebDB::Open(std::string_view args_json) {
         auto db = make_shared_ptr<duckdb::DuckDB>(config_->path, &db_config);
 #ifndef WASM_LOADABLE_EXTENSIONS
         duckdb_web_parquet_init(db.get());
-        duckdb_web_fts_init(db.get());
 #if defined(DUCKDB_JSON_EXTENSION)
         duckdb_web_json_init(db.get());
 #endif

packages/duckdb-wasm/test/fts.test.ts

@@ -21,6 +21,8 @@ export function testFTS(db: () => duckdb.DuckDBBindings): void {
             await conn.query(
                 "INSERT INTO documents VALUES ('doc1', 'The mallard is a dabbling duck that breeds throughout the temperate.','Hannes Mühleisen', 3), ('doc2', 'The cat is a domestic species of small carnivorous mammal.', 'Laurens Kuiper', 2);",
             );
+            // Skip for now, not supported in the no-extension version
+            return;
             await conn.query("PRAGMA create_fts_index('documents', 'document_identifier', 'text_content', 'author');");
             const result = conn.query(
                 'SELECT document_identifier, score\n' +

patches/duckdb/duckdb_smaller_binary_no_select.patch

@@ -0,0 +1,37 @@
+diff --git a/src/execution/expression_executor.cpp b/src/execution/expression_executor.cpp
+index 63a24f479e..eadf1dea0d 100644
+--- a/src/execution/expression_executor.cpp
++++ b/src/execution/expression_executor.cpp
+@@ -236,7 +236,7 @@ idx_t ExpressionExecutor::Select(const Expression &expr, ExpressionState *state,
+ 	D_ASSERT(true_sel || false_sel);
+ 	D_ASSERT(expr.return_type.id() == LogicalTypeId::BOOLEAN);
+	switch (expr.GetExpressionClass()) {
+-#ifndef DUCKDB_SMALLER_BINARY
++#ifndef DUCKDB_SMALLER_BINARY_NO_SELECT
+ 	case ExpressionClass::BOUND_BETWEEN:
+ 		return Select(expr.Cast<BoundBetweenExpression>(), state, sel, count, true_sel, false_sel);
+ #endif
+diff --git a/src/execution/expression_executor/execute_between.cpp b/src/execution/expression_executor/execute_between.cpp
+index 3418351362..52cf454530 100644
+--- a/src/execution/expression_executor/execute_between.cpp
++++ b/src/execution/expression_executor/execute_between.cpp
+@@ -7,7 +7,7 @@
+
+ namespace duckdb {
+
+-#ifndef DUCKDB_SMALLER_BINARY
++#ifndef DUCKDB_SMALLER_BINARY_NO_SELECT
+ struct BothInclusiveBetweenOperator {
+ 	template <class T>
+ 	static inline bool Operation(T input, T lower, T upper) {
+@@ -133,8 +133,8 @@ void ExpressionExecutor::Execute(const BoundBetweenExpression &expr, ExpressionS
+
+ idx_t ExpressionExecutor::Select(const BoundBetweenExpression &expr, ExpressionState *state, const SelectionVector *sel,
+                                  idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) {
+-#ifdef DUCKDB_SMALLER_BINARY
+-	throw InternalException("ExpressionExecutor::Select not available with DUCKDB_SMALLER_BINARY");
++#ifdef DUCKDB_SMALLER_BINARY_NO_SELECT
++	throw InternalException("ExpressionExecutor::Select not available with DUCKDB_SMALLER_BINARY_NO_SELECT");
+ #else
+ 	// resolve the children
+ 	Vector input(state->intermediate_chunk.data[0]);