Update vendored DuckDB sources to d573b275ce

Author: duckdblabs-bot

src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -105,7 +105,6 @@ struct ParquetOptions {
 	explicit ParquetOptions(ClientContext &context);
 
 	bool binary_as_string = false;
-	bool variant_legacy_encoding = false;
 	bool file_row_number = false;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
 	bool debug_use_openssl = true;

src/duckdb/extension/parquet/include/reader/variant/variant_value.hpp

@@ -42,6 +42,7 @@ struct VariantValue {
 
 public:
 	yyjson_mut_val *ToJSON(ClientContext &context, yyjson_mut_doc *doc) const;
+	static void ToVARIANT(vector<VariantValue> &input, Vector &result);
 
 public:
 	VariantValueType value_type;

src/duckdb/extension/parquet/include/reader/variant_column_reader.hpp

@@ -15,7 +15,7 @@ namespace duckdb {
 
 class VariantColumnReader : public ColumnReader {
 public:
-	static constexpr const PhysicalType TYPE = PhysicalType::VARCHAR;
+	static constexpr const PhysicalType TYPE = PhysicalType::STRUCT;
 
 public:
 	VariantColumnReader(ClientContext &context, ParquetReader &reader, const ParquetColumnSchema &schema,

src/duckdb/extension/parquet/parquet_extension.cpp

@@ -998,9 +998,6 @@ static void LoadInternal(ExtensionLoader &loader) {
 	    "enable_geoparquet_conversion",
 	    "Attempt to decode/encode geometry data in/as GeoParquet files if the spatial extension is present.",
 	    LogicalType::BOOLEAN, Value::BOOLEAN(true));
-	config.AddExtensionOption("variant_legacy_encoding",
-	                          "Enables the Parquet reader to identify a Variant structurally.", LogicalType::BOOLEAN,
-	                          Value::BOOLEAN(false));
 }
 
 void ParquetExtension::Load(ExtensionLoader &loader) {

src/duckdb/extension/parquet/parquet_multi_file_info.cpp

@@ -397,10 +397,6 @@ bool ParquetMultiFileInfo::ParseOption(ClientContext &context, const string &ori
 		options.binary_as_string = BooleanValue::Get(val);
 		return true;
 	}
-	if (key == "variant_legacy_encoding") {
-		options.variant_legacy_encoding = BooleanValue::Get(val);
-		return true;
-	}
 	if (key == "file_row_number") {
 		options.file_row_number = BooleanValue::Get(val);
 		return true;

src/duckdb/extension/parquet/parquet_reader.cpp

@@ -517,52 +517,6 @@ unique_ptr<BaseStatistics> ParquetColumnSchema::Stats(const FileMetaData &file_m
 	return ParquetStatisticsUtils::TransformColumnStatistics(*this, columns, parquet_options.can_have_nan);
 }
 
-static bool IsVariantType(const SchemaElement &root, const vector<ParquetColumnSchema> &children) {
-	if (children.size() < 2) {
-		return false;
-	}
-	auto &child0 = children[0];
-	auto &child1 = children[1];
-
-	ParquetColumnSchema const *metadata;
-	ParquetColumnSchema const *value;
-
-	if (child0.name == "metadata" && child1.name == "value") {
-		metadata = &child0;
-		value = &child1;
-	} else if (child1.name == "metadata" && child0.name == "value") {
-		metadata = &child1;
-		value = &child0;
-	} else {
-		return false;
-	}
-
-	//! Verify names
-	if (metadata->name != "metadata") {
-		return false;
-	}
-	if (value->name != "value") {
-		return false;
-	}
-
-	//! Verify types
-	if (metadata->parquet_type != duckdb_parquet::Type::BYTE_ARRAY) {
-		return false;
-	}
-	if (value->parquet_type != duckdb_parquet::Type::BYTE_ARRAY) {
-		return false;
-	}
-	if (children.size() == 3) {
-		auto &typed_value = children[2];
-		if (typed_value.name != "typed_value") {
-			return false;
-		}
-	} else if (children.size() != 2) {
-		return false;
-	}
-	return true;
-}
-
 ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_define, idx_t max_repeat,
                                                         idx_t &next_schema_idx, idx_t &next_file_idx,
                                                         ClientContext &context) {
@@ -629,9 +583,6 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 		const bool is_map = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP;
 		bool is_map_kv = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP_KEY_VALUE;
 		bool is_variant = s_ele.__isset.logicalType && s_ele.logicalType.__isset.VARIANT == true;
-		if (!is_variant) {
-			is_variant = parquet_options.variant_legacy_encoding && IsVariantType(s_ele, child_schemas);
-		}
 
 		if (!is_map_kv && this_idx > 0) {
 			// check if the parent node of this is a map
@@ -667,7 +618,7 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 
 			LogicalType result_type;
 			if (is_variant) {
-				result_type = LogicalType::JSON();
+				result_type = LogicalType::VARIANT();
 			} else {
 				result_type = LogicalType::STRUCT(std::move(struct_types));
 			}
@@ -808,9 +759,6 @@ ParquetOptions::ParquetOptions(ClientContext &context) {
 	if (context.TryGetCurrentSetting("binary_as_string", lookup_value)) {
 		binary_as_string = lookup_value.GetValue<bool>();
 	}
-	if (context.TryGetCurrentSetting("variant_legacy_encoding", lookup_value)) {
-		variant_legacy_encoding = lookup_value.GetValue<bool>();
-	}
 }
 
 ParquetColumnDefinition ParquetColumnDefinition::FromSchemaValue(ClientContext &context, const Value &column_value) {

src/duckdb/extension/parquet/parquet_writer.cpp

@@ -541,7 +541,7 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 	row_group.__isset.total_compressed_size = true;
 
 	if (encryption_config) {
-		auto row_group_ordinal = num_row_groups.load();
+		const auto row_group_ordinal = file_meta_data.row_groups.size();
 		if (row_group_ordinal > std::numeric_limits<int16_t>::max()) {
 			throw InvalidInputException("RowGroup ordinal exceeds 32767 when encryption enabled");
 		}
@@ -562,6 +562,14 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 		return;
 	}
 
+	// "total_written" is only used for the FILE_SIZE_BYTES flag, and only when threads are writing in parallel.
+	// We pre-emptively increase it here to try to reduce overshooting when many threads are writing in parallel.
+	// However, waiting for the exact value (PrepareRowGroup) takes too long, and would cause overshoots to happen.
+	// So, we guess the compression ratio. We guess 3x, but this will be off depending on the data.
+	// "total_written" is restored to the exact number of written bytes at the end of FlushRowGroup.
+	// PhysicalCopyToFile should be reworked to use prepare/flush batch separately for better accuracy.
+	total_written += buffer.SizeInBytes() / 2;
+
 	PreparedRowGroup prepared_row_group;
 	PrepareRowGroup(buffer, prepared_row_group);
 	buffer.Reset();