Update vendored DuckDB sources to 5511fb39ac

Author: duckdblabs-bot

src/duckdb/extension/core_functions/function_list.cpp

@@ -158,7 +158,7 @@ static const StaticFunctionDefinition core_functions[] = {
 	DUCKDB_SCALAR_FUNCTION_SET(DayOfWeekFun),
 	DUCKDB_SCALAR_FUNCTION_SET(DayOfYearFun),
 	DUCKDB_SCALAR_FUNCTION_SET(DecadeFun),
-	DUCKDB_SCALAR_FUNCTION(DecodeFun),
+	DUCKDB_SCALAR_FUNCTION_SET(DecodeFun),
 	DUCKDB_SCALAR_FUNCTION(DegreesFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(Editdist3Fun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ElementAtFun),

src/duckdb/extension/core_functions/include/core_functions/scalar/blob_functions.hpp

@@ -17,12 +17,12 @@ namespace duckdb {
 
 struct DecodeFun {
 	static constexpr const char *Name = "decode";
-	static constexpr const char *Parameters = "blob";
-	static constexpr const char *Description = "Converts `blob` to `VARCHAR`. Fails if `blob` is not valid UTF-8.";
-	static constexpr const char *Example = "decode('\\xC3\\xBC'::BLOB)";
+	static constexpr const char *Parameters = "blob,varchar";
+	static constexpr const char *Description = "Converts `blob` to `VARCHAR`. Invalid UTF-8 is handled based on the error behavior argument. Can be 'strict' (default, fail), 'replace' to replace invalid characters with '?', or 'ignore' to skip invalid characters.";
+	static constexpr const char *Example = "decode('\\xC3\\xBC'::BLOB)\002decode('\\xA0'::BLOB, 'replace')\002decode('\\xA0'::BLOB, 'ignore')";
 	static constexpr const char *Categories = "blob";
 
-	static ScalarFunction GetFunction();
+	static ScalarFunctionSet GetFunctions();
 };
 
 struct EncodeFun {

src/duckdb/extension/core_functions/scalar/blob/encode.cpp

@@ -1,5 +1,6 @@
 #include "core_functions/scalar/blob_functions.hpp"
 #include "utf8proc_wrapper.hpp"
+#include "duckdb/common/string_util.hpp"
 #include "duckdb/common/exception/conversion_exception.hpp"
 
 namespace duckdb {
@@ -12,22 +13,87 @@ void EncodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	result.Reinterpret(args.data[0]);
 }
 
-struct BlobDecodeOperator {
+enum class DecodeErrorBehavior : uint8_t {
+	STRICT = 1,  // raise an error
+	REPLACE = 2, // replace invalid characters with '?'
+	IGNORE = 3   // ignore invalid characters (remove from string)
+};
+
+DecodeErrorBehavior GetDecodeErrorBehavior(const string_t &specifier_p) {
+	auto size = specifier_p.GetSize();
+	auto data = specifier_p.GetData();
+	if (StringUtil::CIEquals(data, size, "strict", 6)) {
+		return DecodeErrorBehavior::STRICT;
+	} else if (StringUtil::CIEquals(data, size, "replace", 7)) {
+		return DecodeErrorBehavior::REPLACE;
+	} else if (StringUtil::CIEquals(data, size, "ignore", 6)) {
+		return DecodeErrorBehavior::IGNORE;
+	} else {
+		throw ConversionException("decode error behavior specifier \"%s\" not recognized", specifier_p.GetString());
+	}
+}
+
+struct UnaryBlobDecodeOperator {
 	template <class INPUT_TYPE, class RESULT_TYPE>
 	static RESULT_TYPE Operation(INPUT_TYPE input) {
 		auto input_data = input.GetData();
 		auto input_length = input.GetSize();
 		if (Utf8Proc::Analyze(input_data, input_length) == UnicodeType::INVALID) {
 			throw ConversionException(
-			    "Failure in decode: could not convert blob to UTF8 string, the blob contained invalid UTF8 characters");
+			    "Failure in decode: could not convert blob to UTF8 string, the blob "
+			    "contained invalid UTF8 characters. \n"
+			    "Use try(decode(BLOB)) to return NULL and continue instead of returning an error. "
+			    "Specify decode(BLOB, 'replace') to replace invalid characters with '?'. "
+			    "Specify decode(BLOB, 'ignore') to remove invalid characters when encountered.");
 		}
 		return input;
 	}
 };
 
-void DecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+void UnaryDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	// decode is also a nop cast, but requires verification if the provided string is actually
+	UnaryExecutor::Execute<string_t, string_t, UnaryBlobDecodeOperator>(args.data[0], result, args.size());
+	StringVector::AddHeapReference(result, args.data[0]);
+}
+
+void BinaryDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	// decode is also a nop cast, but requires verification if the provided string is actually
-	UnaryExecutor::Execute<string_t, string_t, BlobDecodeOperator>(args.data[0], result, args.size());
+	BinaryExecutor::Execute<string_t, string_t, string_t>(
+	    args.data[0], args.data[1], result, args.size(), [&](string_t input, string_t error_option) {
+		    auto input_data = input.GetDataWriteable();
+		    auto input_length = input.GetSize();
+
+		    if (Utf8Proc::Analyze(input_data, input_length) != UnicodeType::INVALID) {
+			    return input;
+		    }
+		    auto const error_behavior = GetDecodeErrorBehavior(error_option);
+
+		    switch (error_behavior) {
+		    case DecodeErrorBehavior::REPLACE:
+			    Utf8Proc::MakeValid(input_data, input_length);
+			    return input;
+
+		    case DecodeErrorBehavior::IGNORE: {
+			    auto new_str = Utf8Proc::RemoveInvalid(input_data, input_length);
+			    auto target = StringVector::EmptyString(result, new_str.size());
+			    auto output = target.GetDataWriteable();
+			    memcpy(output, new_str.data(), new_str.size());
+			    target.Finalize();
+			    return target;
+		    }
+
+		    case DecodeErrorBehavior::STRICT:
+			    throw ConversionException(
+			        "Failure in decode: could not convert blob to UTF8 string, the blob "
+			        "contained invalid UTF8 characters. \n"
+			        "Use try(decode(BLOB)) to return NULL and continue instead of returning an error. "
+			        "Specify decode(BLOB, 'replace') to replace invalid characters with '?'. "
+			        "Specify decode(BLOB, 'ignore') to remove invalid characters when encountered.");
+
+		    default:
+			    throw InternalException("Unimplemented decode error behavior");
+		    }
+	    });
 	StringVector::AddHeapReference(result, args.data[0]);
 }
 
@@ -37,10 +103,20 @@ ScalarFunction EncodeFun::GetFunction() {
 	return ScalarFunction({LogicalType::VARCHAR}, LogicalType::BLOB, EncodeFunction);
 }
 
-ScalarFunction DecodeFun::GetFunction() {
-	ScalarFunction function({LogicalType::BLOB}, LogicalType::VARCHAR, DecodeFunction);
-	function.SetFallible();
-	return function;
+ScalarFunctionSet DecodeFun::GetFunctions() {
+	ScalarFunctionSet decode("decode");
+
+	ScalarFunction unary_function({LogicalType::BLOB}, LogicalType::VARCHAR, UnaryDecodeFunction);
+	ScalarFunction binary_function({LogicalType::BLOB, LogicalType::VARCHAR}, LogicalType::VARCHAR,
+	                               BinaryDecodeFunction);
+
+	unary_function.SetFallible();
+	binary_function.SetFallible();
+
+	decode.AddFunction(unary_function);
+	decode.AddFunction(binary_function);
+
+	return decode;
 }
 
 } // namespace duckdb

src/duckdb/src/catalog/default/default_functions.cpp

@@ -115,10 +115,6 @@ static const DefaultMacro internal_macros[] = {
     {DEFAULT_SCHEMA, "list_reverse", {"l", nullptr}, {{nullptr, nullptr}}, "l[:-:-1]"},
     {DEFAULT_SCHEMA, "array_reverse", {"l", nullptr}, {{nullptr, nullptr}}, "list_reverse(l)"},
 
-    // FIXME implement as actual function if we encounter a lot of performance issues. Complexity now: n * m, with hashing possibly n + m
-    {DEFAULT_SCHEMA, "list_intersect", {"l1", "l2", nullptr}, {{nullptr, nullptr}}, "list_filter(list_distinct(l1), lambda variable_intersect: list_contains(l2, variable_intersect))"},
-    {DEFAULT_SCHEMA, "array_intersect", {"l1", "l2", nullptr}, {{nullptr, nullptr}}, "list_intersect(l1, l2)"},
-
 	// algebraic list aggregates
 	{DEFAULT_SCHEMA, "list_avg", {"l", nullptr}, {{nullptr, nullptr}}, "list_aggr(l, 'avg')"},
 	{DEFAULT_SCHEMA, "list_var_samp", {"l", nullptr}, {{nullptr, nullptr}}, "list_aggr(l, 'var_samp')"},

src/duckdb/src/common/enum_util.cpp

@@ -3812,19 +3812,20 @@ const StringUtil::EnumStringLiteral *GetRelationTypeValues() {
 		{ static_cast<uint32_t>(RelationType::VIEW_RELATION), "VIEW_RELATION" },
 		{ static_cast<uint32_t>(RelationType::QUERY_RELATION), "QUERY_RELATION" },
 		{ static_cast<uint32_t>(RelationType::DELIM_JOIN_RELATION), "DELIM_JOIN_RELATION" },
-		{ static_cast<uint32_t>(RelationType::DELIM_GET_RELATION), "DELIM_GET_RELATION" }
+		{ static_cast<uint32_t>(RelationType::DELIM_GET_RELATION), "DELIM_GET_RELATION" },
+		{ static_cast<uint32_t>(RelationType::EXTENSION_RELATION), "EXTENSION_RELATION" }
 	};
 	return values;
 }
 
 template<>
 const char* EnumUtil::ToChars<RelationType>(RelationType value) {
-	return StringUtil::EnumToString(GetRelationTypeValues(), 28, "RelationType", static_cast<uint32_t>(value));
+	return StringUtil::EnumToString(GetRelationTypeValues(), 29, "RelationType", static_cast<uint32_t>(value));
 }
 
 template<>
 RelationType EnumUtil::FromString<RelationType>(const char *value) {
-	return static_cast<RelationType>(StringUtil::StringToEnum(GetRelationTypeValues(), 28, "RelationType", value));
+	return static_cast<RelationType>(StringUtil::StringToEnum(GetRelationTypeValues(), 29, "RelationType", value));
 }
 
 const StringUtil::EnumStringLiteral *GetRenderModeValues() {

src/duckdb/src/common/enums/relation_type.cpp

@@ -61,6 +61,8 @@ string RelationTypeToString(RelationType type) {
 		return "VIEW_RELATION";
 	case RelationType::QUERY_RELATION:
 		return "QUERY_RELATION";
+	case RelationType::EXTENSION_RELATION:
+		return "EXTENSION_RELATION";
 	case RelationType::INVALID_RELATION:
 		break;
 	}

src/duckdb/src/common/sort/hashed_sort.cpp

@@ -759,6 +759,16 @@ static SourceResultType MaterializeHashGroupData(ExecutionContext &context, idx_
 		return hash_group.get_columns++ ? SourceResultType::HAVE_MORE_OUTPUT : SourceResultType::FINISHED;
 	}
 
+	//	OVER(PARTITION BY...)
+	if (gsink.grouping_data) {
+		lock_guard<mutex> reset_guard(hash_group.scan_lock);
+		auto &partitions = gsink.grouping_data->GetPartitions();
+		if (hash_bin < partitions.size()) {
+			//	Release the memory now that we have finished scanning it.
+			partitions[hash_bin].reset();
+		}
+	}
+
 	auto &sort = *hash_group.sort;
 	auto &sort_global = *hash_group.sort_source;
 	auto sort_local = sort.GetLocalSourceState(context, sort_global);
@@ -792,6 +802,7 @@ HashedSort::HashGroupPtr HashedSort::GetColumnData(idx_t hash_bin, OperatorSourc
 
 	OperatorSourceInput input {sort_global, source.local_state, source.interrupt_state};
 	auto result = sort.GetColumnData(input);
+	hash_group.sort_source.reset();
 
 	//	Just because MaterializeColumnData returned FINISHED doesn't mean that the same thread will
 	//	get the result...
@@ -824,6 +835,8 @@ HashedSort::SortedRunPtr HashedSort::GetSortedRun(ClientContext &client, idx_t h
 		result = make_uniq<SortedRun>(client, sort, false);
 	}
 
+	hash_group.sort_source.reset();
+
 	return result;
 }
 

src/duckdb/src/execution/operator/join/physical_asof_join.cpp

@@ -1303,10 +1303,6 @@ class AsOfLocalSourceState : public LocalSourceState {
 		case AsOfJoinSourceStage::DONE:
 			throw InternalException("Invalid state for AsOf Task");
 		}
-
-		if (TaskFinished()) {
-			++gsource.finished;
-		}
 	}
 
 	void BeginRightScan();
@@ -1392,6 +1388,8 @@ void AsOfGlobalSourceState::FinishTask(TaskPtr task) {
 		return;
 	}
 
+	++finished;
+
 	const auto group_idx = task->group_idx;
 	auto &finished_hash_group = asof_groups[group_idx];
 	D_ASSERT(finished_hash_group);
@@ -1587,7 +1585,12 @@ SourceResultType PhysicalAsOfJoin::GetData(ExecutionContext &context, DataChunk
 	// Therefore, we loop until we've produced tuples, or until the operator is actually done
 	while (gsource.HasUnfinishedTasks() && chunk.size() == 0) {
 		if (!lsource.TaskFinished() || lsource.TryAssignTask()) {
-			lsource.ExecuteTask(context, chunk, input);
+			try {
+				lsource.ExecuteTask(context, chunk, input);
+			} catch (...) {
+				gsource.stopped = true;
+				throw;
+			}
 		} else {
 			auto guard = gsource.Lock();
 			if (!gsource.HasMoreTasks()) {

src/duckdb/src/function/function_list.cpp

@@ -81,6 +81,7 @@ static const StaticFunctionDefinition function[] = {
 	DUCKDB_SCALAR_FUNCTION_SET(ArrayExtractFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayHasFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayIndexofFun),
+	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayIntersectFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ArrayLengthFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayPositionFun),
 	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArrayResizeFun),
@@ -122,6 +123,7 @@ static const StaticFunctionDefinition function[] = {
 	DUCKDB_SCALAR_FUNCTION_SET(ListExtractFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListHasFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListIndexofFun),
+	DUCKDB_SCALAR_FUNCTION(ListIntersectFun),
 	DUCKDB_SCALAR_FUNCTION(ListPositionFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListResizeFun),
 	DUCKDB_SCALAR_FUNCTION(ListSelectFun),