diff --git a/datafusion-testing b/datafusion-testing index 243047b9dd68..3462eaa78745 160000 --- a/datafusion-testing +++ b/datafusion-testing @@ -1 +1 @@ -Subproject commit 243047b9dd682be688628539c604daaddfe640f9 +Subproject commit 3462eaa787459957e38df267a4a21f5bea605807 diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rs index 39c79b4b9974..33b72f68bc05 100644 --- a/datafusion/common/src/types/native.rs +++ b/datafusion/common/src/types/native.rs @@ -436,6 +436,12 @@ impl NativeType { ) } + #[inline] + pub fn is_float(&self) -> bool { + use NativeType::*; + matches!(self, Float16 | Float32 | Float64) + } + #[inline] pub fn is_integer(&self) -> bool { use NativeType::*; diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index 063417a254be..324865c77181 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -215,8 +215,8 @@ pub enum TypeSignatureClass { Interval, Duration, Native(LogicalTypeRef), - // TODO: - // Numeric + Numeric, + Float, Integer, } @@ -252,6 +252,16 @@ impl TypeSignatureClass { TypeSignatureClass::Duration => { vec![DataType::Duration(TimeUnit::Nanosecond)] } + TypeSignatureClass::Numeric => { + vec![ + DataType::Int64, + DataType::Float64, + DataType::Decimal256(3, -2), + ] + } + TypeSignatureClass::Float => { + vec![DataType::Float64] + } TypeSignatureClass::Integer => { vec![DataType::Int64] } @@ -263,16 +273,14 @@ impl TypeSignatureClass { self: &TypeSignatureClass, logical_type: &NativeType, ) -> bool { - if logical_type == &NativeType::Null { - return true; - } - match self { TypeSignatureClass::Native(t) if t.native() == logical_type => true, TypeSignatureClass::Timestamp if logical_type.is_timestamp() => true, TypeSignatureClass::Time if logical_type.is_time() => true, TypeSignatureClass::Interval if logical_type.is_interval() => true, TypeSignatureClass::Duration if logical_type.is_duration() => true, + TypeSignatureClass::Numeric if logical_type.is_numeric() => true, + TypeSignatureClass::Float if logical_type.is_float() => true, TypeSignatureClass::Integer if logical_type.is_integer() => true, _ => false, } @@ -301,6 +309,12 @@ impl TypeSignatureClass { TypeSignatureClass::Duration if native_type.is_duration() => { Ok(origin_type.to_owned()) } + TypeSignatureClass::Numeric if native_type.is_numeric() => { + Ok(origin_type.to_owned()) + } + TypeSignatureClass::Float if native_type.is_float() => { + Ok(origin_type.to_owned()) + } TypeSignatureClass::Integer if native_type.is_integer() => { Ok(origin_type.to_owned()) } diff --git a/datafusion/functions-nested/src/string.rs b/datafusion/functions-nested/src/string.rs index d60d1a6e4de0..7ffa684a83a4 100644 --- a/datafusion/functions-nested/src/string.rs +++ b/datafusion/functions-nested/src/string.rs @@ -43,7 +43,7 @@ use arrow::datatypes::DataType::{ }; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::exec_err; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -255,11 +255,19 @@ impl StringToArray { vec![ TypeSignature::Coercible(vec![ Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), TypeSignature::Coercible(vec![ Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), Coercion::new_exact(TypeSignatureClass::Native(logical_string())), ]), ], diff --git a/datafusion/functions/src/crypto/digest.rs b/datafusion/functions/src/crypto/digest.rs index 2840006169be..7ddc3ad3cd93 100644 --- a/datafusion/functions/src/crypto/digest.rs +++ b/datafusion/functions/src/crypto/digest.rs @@ -19,7 +19,7 @@ use super::basic::{digest, utf8_or_binary_to_binary_type}; use arrow::datatypes::DataType; use datafusion_common::{ - types::{logical_binary, logical_string}, + types::{logical_binary, logical_null, logical_string, NativeType}, Result, }; use datafusion_expr::{ @@ -72,12 +72,28 @@ impl DigestFunc { signature: Signature::one_of( vec![ TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_binary())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_binary()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::Binary, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), ], Volatility::Immutable, diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rs index c1540450029c..3af0550cb8cd 100644 --- a/datafusion/functions/src/crypto/md5.rs +++ b/datafusion/functions/src/crypto/md5.rs @@ -20,7 +20,7 @@ use crate::crypto::basic::md5; use arrow::datatypes::DataType; use datafusion_common::{ plan_err, - types::{logical_binary, logical_string, NativeType}, + types::{logical_binary, logical_null, logical_string, NativeType}, Result, }; use datafusion_expr::{ @@ -62,12 +62,18 @@ impl Md5Func { vec![ TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_string())], + vec![ + TypeSignatureClass::Native(logical_string()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::String, )]), TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_binary())], + vec![ + TypeSignatureClass::Native(logical_binary()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::Binary, )]), ], diff --git a/datafusion/functions/src/crypto/sha224.rs b/datafusion/functions/src/crypto/sha224.rs index a64a3ef80319..8b38a75e0af9 100644 --- a/datafusion/functions/src/crypto/sha224.rs +++ b/datafusion/functions/src/crypto/sha224.rs @@ -19,7 +19,7 @@ use super::basic::{sha224, utf8_or_binary_to_binary_type}; use arrow::datatypes::DataType; use datafusion_common::{ - types::{logical_binary, logical_string, NativeType}, + types::{logical_binary, logical_null, logical_string, NativeType}, Result, }; use datafusion_expr::{ @@ -62,12 +62,18 @@ impl SHA224Func { vec![ TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_string())], + vec![ + TypeSignatureClass::Native(logical_string()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::String, )]), TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_binary())], + vec![ + TypeSignatureClass::Native(logical_binary()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::Binary, )]), ], diff --git a/datafusion/functions/src/crypto/sha256.rs b/datafusion/functions/src/crypto/sha256.rs index 94f3ea3b49fa..89d4b1071f27 100644 --- a/datafusion/functions/src/crypto/sha256.rs +++ b/datafusion/functions/src/crypto/sha256.rs @@ -19,7 +19,7 @@ use super::basic::{sha256, utf8_or_binary_to_binary_type}; use arrow::datatypes::DataType; use datafusion_common::{ - types::{logical_binary, logical_string, NativeType}, + types::{logical_binary, logical_null, logical_string, NativeType}, Result, }; use datafusion_expr::{ @@ -61,12 +61,18 @@ impl SHA256Func { vec![ TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_string())], + vec![ + TypeSignatureClass::Native(logical_string()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::String, )]), TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_binary())], + vec![ + TypeSignatureClass::Native(logical_binary()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::Binary, )]), ], diff --git a/datafusion/functions/src/crypto/sha384.rs b/datafusion/functions/src/crypto/sha384.rs index 023730469c7b..5de5657227b5 100644 --- a/datafusion/functions/src/crypto/sha384.rs +++ b/datafusion/functions/src/crypto/sha384.rs @@ -19,7 +19,7 @@ use super::basic::{sha384, utf8_or_binary_to_binary_type}; use arrow::datatypes::DataType; use datafusion_common::{ - types::{logical_binary, logical_string, NativeType}, + types::{logical_binary, logical_null, logical_string, NativeType}, Result, }; use datafusion_expr::{ @@ -61,12 +61,18 @@ impl SHA384Func { vec![ TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_string())], + vec![ + TypeSignatureClass::Native(logical_string()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::String, )]), TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_binary())], + vec![ + TypeSignatureClass::Native(logical_binary()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::Binary, )]), ], diff --git a/datafusion/functions/src/crypto/sha512.rs b/datafusion/functions/src/crypto/sha512.rs index f48737e5751f..f423dc88a9d7 100644 --- a/datafusion/functions/src/crypto/sha512.rs +++ b/datafusion/functions/src/crypto/sha512.rs @@ -19,7 +19,7 @@ use super::basic::{sha512, utf8_or_binary_to_binary_type}; use arrow::datatypes::DataType; use datafusion_common::{ - types::{logical_binary, logical_string, NativeType}, + types::{logical_binary, logical_null, logical_string, NativeType}, Result, }; use datafusion_expr::{ @@ -61,12 +61,18 @@ impl SHA512Func { vec![ TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_string())], + vec![ + TypeSignatureClass::Native(logical_string()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::String, )]), TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_binary())], + vec![ + TypeSignatureClass::Native(logical_binary()), + TypeSignatureClass::Native(logical_null()), + ], NativeType::Binary, )]), ], diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rs index d2849c3abba0..12bc5649a5a3 100644 --- a/datafusion/functions/src/macros.rs +++ b/datafusion/functions/src/macros.rs @@ -160,13 +160,16 @@ macro_rules! make_math_unary_udf { use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; + use datafusion_common::types::logical_null; + use datafusion_common::types::NativeType; use datafusion_common::{exec_err, Result}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, - Signature, Volatility, + Signature, TypeSignatureClass, Volatility, }; + use datafusion_expr_common::signature::Coercion; #[derive(Debug)] pub struct $UDF { @@ -175,11 +178,16 @@ macro_rules! make_math_unary_udf { impl $UDF { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::uniform( - 1, - vec![Float64, Float32], + signature: Signature::coercible( + vec![Coercion::new_implicit( + TypeSignatureClass::Float, + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/abs.rs b/datafusion/functions/src/math/abs.rs index 0c686a59016a..d2718201d2bb 100644 --- a/datafusion/functions/src/math/abs.rs +++ b/datafusion/functions/src/math/abs.rs @@ -26,6 +26,7 @@ use arrow::array::{ }; use arrow::datatypes::DataType; use arrow::error::ArrowError; +use datafusion_common::types::{logical_null, NativeType}; use datafusion_common::{ internal_datafusion_err, not_impl_err, utils::take_function_args, Result, }; @@ -33,8 +34,9 @@ use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignature, TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; type MathArrayFunction = fn(&ArrayRef) -> Result; @@ -126,7 +128,14 @@ impl Default for AbsFunc { impl AbsFunc { pub fn new() -> Self { Self { - signature: Signature::numeric(1, Volatility::Immutable), + signature: Signature::new( + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Numeric, + vec![TypeSignatureClass::Native(logical_null())], + NativeType::Float64, + )]), + Volatility::Immutable, + ), } } } diff --git a/datafusion/functions/src/math/cot.rs b/datafusion/functions/src/math/cot.rs index 4e56212ddbee..c0b14ab9b9f0 100644 --- a/datafusion/functions/src/math/cot.rs +++ b/datafusion/functions/src/math/cot.rs @@ -21,10 +21,14 @@ use std::sync::Arc; use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::NativeType; +use datafusion_expr_common::signature::Coercion; use crate::utils::make_scalar_function; use datafusion_common::{exec_err, Result}; -use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, TypeSignature, TypeSignatureClass, +}; use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; use datafusion_macros::user_doc; @@ -47,16 +51,18 @@ impl Default for CotFunc { impl CotFunc { pub fn new() -> Self { - use DataType::*; Self { // math expressions expect 1 argument of type f64 or f32 // priority is given to f64 because e.g. `sqrt(1i32)` is in IR (real numbers) and thus we // return the best approximation for it (in f64). // We accept f32 because in this case it is clear that the best approximation // will be as good as the number of digits in the number - signature: Signature::uniform( - 1, - vec![Float64, Float32], + signature: Signature::new( + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Float, + vec![TypeSignatureClass::Integer], + NativeType::Float64, + )]), Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/factorial.rs b/datafusion/functions/src/math/factorial.rs index c2ac21b78f21..01c47da4df6a 100644 --- a/datafusion/functions/src/math/factorial.rs +++ b/datafusion/functions/src/math/factorial.rs @@ -19,6 +19,7 @@ use arrow::{ array::{ArrayRef, Int64Array}, error::ArrowError, }; +use datafusion_expr_common::signature::Coercion; use std::any::Any; use std::sync::Arc; @@ -27,11 +28,13 @@ use arrow::datatypes::DataType::Int64; use crate::utils::make_scalar_function; use datafusion_common::{ - arrow_datafusion_err, exec_err, internal_datafusion_err, DataFusionError, Result, + arrow_datafusion_err, exec_err, internal_datafusion_err, + types::{logical_int64, logical_null, NativeType}, + DataFusionError, Result, }; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; @@ -55,7 +58,17 @@ impl Default for FactorialFunc { impl FactorialFunc { pub fn new() -> Self { Self { - signature: Signature::uniform(1, vec![Int64], Volatility::Immutable), + signature: Signature::new( + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + )]), + Volatility::Immutable, + ), } } } diff --git a/datafusion/functions/src/math/gcd.rs b/datafusion/functions/src/math/gcd.rs index 7fe253b4afbc..8faf7afad3fc 100644 --- a/datafusion/functions/src/math/gcd.rs +++ b/datafusion/functions/src/math/gcd.rs @@ -19,6 +19,7 @@ use arrow::array::{new_null_array, ArrayRef, AsArray, Int64Array, PrimitiveArray use arrow::compute::try_binary; use arrow::datatypes::{DataType, Int64Type}; use arrow::error::ArrowError; +use datafusion_common::types::{logical_int64, logical_null, NativeType}; use std::any::Any; use std::mem::swap; use std::sync::Arc; @@ -26,8 +27,9 @@ use std::sync::Arc; use datafusion_common::{exec_err, internal_datafusion_err, Result, ScalarValue}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; #[user_doc( @@ -51,9 +53,25 @@ impl Default for GcdFunc { impl GcdFunc { pub fn new() -> Self { Self { - signature: Signature::uniform( - 2, - vec![DataType::Int64], + signature: Signature::coercible( + vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + ), + ], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/iszero.rs b/datafusion/functions/src/math/iszero.rs index bc12dfb7898e..d240677dfa29 100644 --- a/datafusion/functions/src/math/iszero.rs +++ b/datafusion/functions/src/math/iszero.rs @@ -22,12 +22,14 @@ use arrow::array::{ArrayRef, AsArray, BooleanArray}; use arrow::datatypes::DataType::{Boolean, Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::{logical_null, NativeType}; use datafusion_common::{exec_err, Result}; -use datafusion_expr::TypeSignature::Exact; +use datafusion_expr::TypeSignature::{self}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; use crate::utils::make_scalar_function; @@ -51,10 +53,16 @@ impl Default for IsZeroFunc { impl IsZeroFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::one_of( - vec![Exact(vec![Float32]), Exact(vec![Float64])], + signature: Signature::new( + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Float, + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + )]), Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/lcm.rs b/datafusion/functions/src/math/lcm.rs index fc6bf9461f28..bb0c68110994 100644 --- a/datafusion/functions/src/math/lcm.rs +++ b/datafusion/functions/src/math/lcm.rs @@ -23,13 +23,15 @@ use arrow::datatypes::DataType; use arrow::datatypes::DataType::Int64; use arrow::error::ArrowError; +use datafusion_common::types::{logical_int64, logical_null, NativeType}; use datafusion_common::{ arrow_datafusion_err, exec_err, internal_datafusion_err, DataFusionError, Result, }; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignature, TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; use super::gcd::unsigned_gcd; @@ -55,9 +57,28 @@ impl Default for LcmFunc { impl LcmFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::uniform(2, vec![Int64], Volatility::Immutable), + signature: Signature::new( + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + ), + ]), + Volatility::Immutable, + ), } } } diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rs index fd135f4c5ec0..ed8e6f849927 100644 --- a/datafusion/functions/src/math/log.rs +++ b/datafusion/functions/src/math/log.rs @@ -24,6 +24,9 @@ use super::power::PowerFunc; use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::{ + logical_float32, logical_float64, logical_null, NativeType, +}; use datafusion_common::{ exec_err, internal_err, plan_datafusion_err, plan_err, Result, ScalarValue, }; @@ -32,9 +35,10 @@ use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ lit, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF, - TypeSignature::*, + TypeSignature, TypeSignatureClass, }; use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; #[user_doc( @@ -58,19 +62,47 @@ impl Default for LogFunc { impl LogFunc { pub fn new() -> Self { - use DataType::*; Self { signature: Signature::one_of( vec![ - Exact(vec![Float32]), - Exact(vec![Float64]), - Exact(vec![Float32, Float32]), - Exact(vec![Float64, Float64]), + TypeSignature::Coercible(vec![Self::implicit_float::<32>()]), + TypeSignature::Coercible(vec![Self::implicit_float::<64>()]), + TypeSignature::Coercible(vec![ + Self::implicit_float::<32>(), + Self::implicit_float::<32>(), + ]), + TypeSignature::Coercible(vec![ + Self::implicit_float::<64>(), + Self::implicit_float::<64>(), + ]), ], Volatility::Immutable, ), } } + + fn implicit_float() -> Coercion { + match U { + 32 => Coercion::new_implicit( + TypeSignatureClass::Native(logical_float32()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float32, + ), + 64 => Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![ + TypeSignatureClass::Float, + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + ), + _ => unreachable!("implicit_float only accepts 32 or 64"), + } + } } impl ScalarUDFImpl for LogFunc { diff --git a/datafusion/functions/src/math/nans.rs b/datafusion/functions/src/math/nans.rs index 34a5c2a1c16b..18b052337f2b 100644 --- a/datafusion/functions/src/math/nans.rs +++ b/datafusion/functions/src/math/nans.rs @@ -18,11 +18,15 @@ //! Math function: `isnan()`. use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::{logical_null, NativeType}; use datafusion_common::{exec_err, Result}; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, TypeSignature}; +use datafusion_expr::{ + ColumnarValue, ScalarFunctionArgs, TypeSignature, TypeSignatureClass, +}; use arrow::array::{ArrayRef, AsArray, BooleanArray}; use datafusion_expr::{Documentation, ScalarUDFImpl, Signature, Volatility}; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; use std::any::Any; use std::sync::Arc; @@ -46,13 +50,16 @@ impl Default for IsNanFunc { impl IsNanFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::one_of( - vec![ - TypeSignature::Exact(vec![Float32]), - TypeSignature::Exact(vec![Float64]), - ], + signature: Signature::new( + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Float, + vec![ + TypeSignatureClass::Numeric, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + )]), Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index 9effb82896ee..fec6c9f4bbb9 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -23,12 +23,14 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayRef, AsArray, Float32Array, Float64Array}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::{logical_float64, logical_null, NativeType}; use datafusion_common::{exec_err, DataFusionError, Result}; -use datafusion_expr::TypeSignature::Exact; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; +use datafusion_expr::{TypeSignature, TypeSignatureClass}; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; #[user_doc( @@ -58,10 +60,28 @@ impl Default for NanvlFunc { impl NanvlFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::one_of( - vec![Exact(vec![Float32, Float32]), Exact(vec![Float64, Float64])], + signature: Signature::new( + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![ + TypeSignatureClass::Float, + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![ + TypeSignatureClass::Float, + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + ), + ]), Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/power.rs b/datafusion/functions/src/math/power.rs index 028ec2fef793..792db3788d8a 100644 --- a/datafusion/functions/src/math/power.rs +++ b/datafusion/functions/src/math/power.rs @@ -23,6 +23,9 @@ use super::log::LogFunc; use arrow::array::{ArrayRef, AsArray, Int64Array}; use arrow::datatypes::{ArrowNativeTypeOp, DataType, Float64Type}; +use datafusion_common::types::{ + logical_float64, logical_int64, logical_null, NativeType, +}; use datafusion_common::{ arrow_datafusion_err, exec_datafusion_err, exec_err, internal_datafusion_err, plan_datafusion_err, DataFusionError, Result, ScalarValue, @@ -31,8 +34,10 @@ use datafusion_expr::expr::ScalarFunction; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; use datafusion_expr::{ ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF, TypeSignature, + TypeSignatureClass, }; use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; #[user_doc( @@ -56,12 +61,47 @@ impl Default for PowerFunc { impl PowerFunc { pub fn new() -> Self { - use DataType::*; Self { signature: Signature::one_of( vec![ - TypeSignature::Exact(vec![Int64, Int64]), - TypeSignature::Exact(vec![Float64, Float64]), + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Int64, + ), + ]), + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![ + TypeSignatureClass::Float, + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![ + TypeSignatureClass::Float, + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + ), + ]), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/math/round.rs b/datafusion/functions/src/math/round.rs index fc87b7e63a62..73102e2adb4b 100644 --- a/datafusion/functions/src/math/round.rs +++ b/datafusion/functions/src/math/round.rs @@ -24,13 +24,14 @@ use arrow::array::{ArrayRef, AsArray, PrimitiveArray}; use arrow::compute::{cast_with_options, CastOptions}; use arrow::datatypes::DataType::{Float32, Float64, Int32}; use arrow::datatypes::{DataType, Float32Type, Float64Type, Int32Type}; +use datafusion_common::types::{logical_null, NativeType}; use datafusion_common::{exec_datafusion_err, exec_err, Result, ScalarValue}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; -use datafusion_expr::TypeSignature::Exact; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignature, TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; #[user_doc( @@ -56,14 +57,25 @@ impl Default for RoundFunc { impl RoundFunc { pub fn new() -> Self { - use DataType::*; Self { signature: Signature::one_of( vec![ - Exact(vec![Float64, Int64]), - Exact(vec![Float32, Int64]), - Exact(vec![Float64]), - Exact(vec![Float32]), + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Numeric, + vec![TypeSignatureClass::Native(logical_null())], + NativeType::Float64, + )]), + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Numeric, + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + ), + Coercion::new_exact(TypeSignatureClass::Integer), + ]), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/math/signum.rs b/datafusion/functions/src/math/signum.rs index ba5422afa768..6d0d1849803c 100644 --- a/datafusion/functions/src/math/signum.rs +++ b/datafusion/functions/src/math/signum.rs @@ -22,12 +22,14 @@ use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::{logical_null, NativeType}; use datafusion_common::{exec_err, Result}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignature, TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; use crate::utils::make_scalar_function; @@ -53,11 +55,16 @@ impl Default for SignumFunc { impl SignumFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::uniform( - 1, - vec![Float64, Float32], + signature: Signature::new( + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Float, + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Native(logical_null()), + ], + NativeType::Float64, + )]), Volatility::Immutable, ), } diff --git a/datafusion/functions/src/math/trunc.rs b/datafusion/functions/src/math/trunc.rs index 2ac291204a0b..c6ce32a64107 100644 --- a/datafusion/functions/src/math/trunc.rs +++ b/datafusion/functions/src/math/trunc.rs @@ -23,14 +23,16 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayRef, AsArray, PrimitiveArray}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type, Int64Type}; +use datafusion_common::types::{logical_null, NativeType}; use datafusion_common::ScalarValue::Int64; use datafusion_common::{exec_err, Result}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; -use datafusion_expr::TypeSignature::Exact; +use datafusion_expr::TypeSignature; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignatureClass, Volatility, }; +use datafusion_expr_common::signature::Coercion; use datafusion_macros::user_doc; #[user_doc( @@ -60,7 +62,6 @@ impl Default for TruncFunc { impl TruncFunc { pub fn new() -> Self { - use DataType::*; Self { // math expressions expect 1 argument of type f64 or f32 // priority is given to f64 because e.g. `sqrt(1i32)` is in IR (real numbers) and thus we @@ -69,10 +70,19 @@ impl TruncFunc { // will be as good as the number of digits in the number signature: Signature::one_of( vec![ - Exact(vec![Float32, Int64]), - Exact(vec![Float64, Int64]), - Exact(vec![Float64]), - Exact(vec![Float32]), + TypeSignature::Coercible(vec![Coercion::new_implicit( + TypeSignatureClass::Float, + vec![TypeSignatureClass::Native(logical_null())], + NativeType::Float64, + )]), + TypeSignature::Coercible(vec![ + Coercion::new_implicit( + TypeSignatureClass::Float, + vec![TypeSignatureClass::Native(logical_null())], + NativeType::Float64, + ), + Coercion::new_exact(TypeSignatureClass::Integer), + ]), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rs index 2080bb9fe818..f2a062c5a197 100644 --- a/datafusion/functions/src/regex/regexplike.rs +++ b/datafusion/functions/src/regex/regexplike.rs @@ -21,7 +21,7 @@ use arrow::array::{Array, ArrayRef, AsArray, GenericStringArray}; use arrow::compute::kernels::regexp; use arrow::datatypes::DataType; use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{ arrow_datafusion_err, exec_err, internal_err, plan_err, DataFusionError, Result, ScalarValue, @@ -84,12 +84,28 @@ impl RegexpLikeFunc { signature: Signature::one_of( vec![ TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), Coercion::new_exact(TypeSignatureClass::Native(logical_string())), ]), ], diff --git a/datafusion/functions/src/string/ascii.rs b/datafusion/functions/src/string/ascii.rs index 006492a0e07a..702148c71550 100644 --- a/datafusion/functions/src/string/ascii.rs +++ b/datafusion/functions/src/string/ascii.rs @@ -19,7 +19,7 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayAccessor, ArrayIter, ArrayRef, AsArray, Int32Array}; use arrow::datatypes::DataType; use arrow::error::ArrowError; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{internal_err, Result}; use datafusion_expr::{ColumnarValue, Documentation, TypeSignatureClass}; use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; @@ -64,9 +64,11 @@ impl AsciiFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/string/bit_length.rs b/datafusion/functions/src/string/bit_length.rs index f8740aa4178b..ad20f3338680 100644 --- a/datafusion/functions/src/string/bit_length.rs +++ b/datafusion/functions/src/string/bit_length.rs @@ -20,7 +20,7 @@ use arrow::datatypes::DataType; use std::any::Any; use crate::utils::utf8_to_int_type; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::{ @@ -60,9 +60,11 @@ impl BitLengthFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/string/btrim.rs b/datafusion/functions/src/string/btrim.rs index 2f1711c9962a..8851b2f24df7 100644 --- a/datafusion/functions/src/string/btrim.rs +++ b/datafusion/functions/src/string/btrim.rs @@ -19,7 +19,7 @@ use crate::string::common::*; use crate::utils::{make_scalar_function, utf8_to_str_type}; use arrow::array::{ArrayRef, OffsetSizeTrait}; use arrow::datatypes::DataType; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, Result}; use datafusion_expr::function::Hint; use datafusion_expr::{ @@ -83,11 +83,21 @@ impl BTrimFunc { signature: Signature::one_of( vec![ TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), - TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, )]), ], Volatility::Immutable, diff --git a/datafusion/functions/src/string/contains.rs b/datafusion/functions/src/string/contains.rs index 05a3edf61c5a..5bf2ef7e33f1 100644 --- a/datafusion/functions/src/string/contains.rs +++ b/datafusion/functions/src/string/contains.rs @@ -20,7 +20,7 @@ use arrow::array::{Array, ArrayRef, AsArray}; use arrow::compute::contains as arrow_contains; use arrow::datatypes::DataType; use arrow::datatypes::DataType::{Boolean, LargeUtf8, Utf8, Utf8View}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, DataFusionError, Result}; use datafusion_expr::binary::{binary_to_string_coercion, string_coercion}; use datafusion_expr::{ @@ -63,7 +63,11 @@ impl ContainsFunc { signature: Signature::coercible( vec![ Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/string/ends_with.rs b/datafusion/functions/src/string/ends_with.rs index eafc310236ee..5939bd46aaed 100644 --- a/datafusion/functions/src/string/ends_with.rs +++ b/datafusion/functions/src/string/ends_with.rs @@ -22,7 +22,7 @@ use arrow::array::ArrayRef; use arrow::datatypes::DataType; use crate::utils::make_scalar_function; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{internal_err, Result}; use datafusion_expr::binary::{binary_to_string_coercion, string_coercion}; use datafusion_expr::{ @@ -68,8 +68,16 @@ impl EndsWithFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/string/levenshtein.rs b/datafusion/functions/src/string/levenshtein.rs index a1a486c7d3cf..211adc9665b2 100644 --- a/datafusion/functions/src/string/levenshtein.rs +++ b/datafusion/functions/src/string/levenshtein.rs @@ -23,7 +23,7 @@ use arrow::datatypes::DataType; use crate::utils::{make_scalar_function, utf8_to_int_type}; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::utils::datafusion_strsim; use datafusion_common::utils::take_function_args; use datafusion_common::{exec_err, Result}; @@ -73,8 +73,16 @@ impl LevenshteinFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/string/lower.rs b/datafusion/functions/src/string/lower.rs index 226275b13999..c7a427768bbd 100644 --- a/datafusion/functions/src/string/lower.rs +++ b/datafusion/functions/src/string/lower.rs @@ -20,7 +20,7 @@ use std::any::Any; use crate::string::common::to_lower; use crate::utils::utf8_to_str_type; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::Result; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -59,9 +59,11 @@ impl LowerFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/string/ltrim.rs b/datafusion/functions/src/string/ltrim.rs index 65849202efc6..30375a961964 100644 --- a/datafusion/functions/src/string/ltrim.rs +++ b/datafusion/functions/src/string/ltrim.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use crate::string::common::*; use crate::utils::{make_scalar_function, utf8_to_str_type}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, Result}; use datafusion_expr::function::Hint; use datafusion_expr::{ @@ -88,11 +88,21 @@ impl LtrimFunc { signature: Signature::one_of( vec![ TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), - TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, )]), ], Volatility::Immutable, diff --git a/datafusion/functions/src/string/octet_length.rs b/datafusion/functions/src/string/octet_length.rs index 17ea2726b071..71c124a823a0 100644 --- a/datafusion/functions/src/string/octet_length.rs +++ b/datafusion/functions/src/string/octet_length.rs @@ -20,7 +20,7 @@ use arrow::datatypes::DataType; use std::any::Any; use crate::utils::utf8_to_int_type; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::{ @@ -60,9 +60,11 @@ impl OctetLengthFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs index 2d36cb8356a0..6f53462b3067 100644 --- a/datafusion/functions/src/string/repeat.rs +++ b/datafusion/functions/src/string/repeat.rs @@ -26,7 +26,7 @@ use arrow::array::{ use arrow::datatypes::DataType; use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View}; use datafusion_common::cast::as_int64_array; -use datafusion_common::types::{logical_int64, logical_string, NativeType}; +use datafusion_common::types::{logical_int64, logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, DataFusionError, Result}; use datafusion_expr::{ColumnarValue, Documentation, Volatility}; use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature}; @@ -67,7 +67,11 @@ impl RepeatFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), // Accept all integer types but cast them to i64 Coercion::new_implicit( TypeSignatureClass::Native(logical_int64()), diff --git a/datafusion/functions/src/string/replace.rs b/datafusion/functions/src/string/replace.rs index de70215c49c7..222bf2c0f328 100644 --- a/datafusion/functions/src/string/replace.rs +++ b/datafusion/functions/src/string/replace.rs @@ -23,7 +23,7 @@ use arrow::datatypes::DataType; use crate::utils::{make_scalar_function, utf8_to_str_type}; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, Result}; use datafusion_expr::type_coercion::binary::{ binary_to_string_coercion, string_coercion, @@ -68,9 +68,21 @@ impl ReplaceFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/string/rtrim.rs b/datafusion/functions/src/string/rtrim.rs index bb33274978da..9aa0be7db4c5 100644 --- a/datafusion/functions/src/string/rtrim.rs +++ b/datafusion/functions/src/string/rtrim.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use crate::string::common::*; use crate::utils::{make_scalar_function, utf8_to_str_type}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, Result}; use datafusion_expr::function::Hint; use datafusion_expr::{ @@ -88,11 +88,21 @@ impl RtrimFunc { signature: Signature::one_of( vec![ TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ]), - TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignature::Coercible(vec![Coercion::new_implicit( TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, )]), ], Volatility::Immutable, diff --git a/datafusion/functions/src/string/starts_with.rs b/datafusion/functions/src/string/starts_with.rs index 71df83352f96..310155fb835d 100644 --- a/datafusion/functions/src/string/starts_with.rs +++ b/datafusion/functions/src/string/starts_with.rs @@ -26,7 +26,7 @@ use datafusion_expr::type_coercion::binary::{ }; use crate::utils::make_scalar_function; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{internal_err, Result, ScalarValue}; use datafusion_expr::{ cast, Coercion, ColumnarValue, Documentation, Expr, Like, ScalarFunctionArgs, @@ -90,8 +90,16 @@ impl StartsWithFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ], Volatility::Immutable, ), diff --git a/datafusion/functions/src/string/upper.rs b/datafusion/functions/src/string/upper.rs index 2fec7305d183..f2d9b5aee502 100644 --- a/datafusion/functions/src/string/upper.rs +++ b/datafusion/functions/src/string/upper.rs @@ -18,7 +18,7 @@ use crate::string::common::to_upper; use crate::utils::utf8_to_str_type; use arrow::datatypes::DataType; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::Result; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -58,9 +58,11 @@ impl UpperFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/unicode/initcap.rs b/datafusion/functions/src/unicode/initcap.rs index c9b0cb77b096..8b8f17846471 100644 --- a/datafusion/functions/src/unicode/initcap.rs +++ b/datafusion/functions/src/unicode/initcap.rs @@ -25,7 +25,7 @@ use arrow::datatypes::DataType; use crate::utils::{make_scalar_function, utf8_to_str_type}; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass, @@ -65,9 +65,11 @@ impl InitcapFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + )], Volatility::Immutable, ), } diff --git a/datafusion/functions/src/unicode/strpos.rs b/datafusion/functions/src/unicode/strpos.rs index b3bc73a29585..7eaf35446492 100644 --- a/datafusion/functions/src/unicode/strpos.rs +++ b/datafusion/functions/src/unicode/strpos.rs @@ -23,7 +23,7 @@ use arrow::array::{ ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray, StringArrayType, }; use arrow::datatypes::{ArrowNativeType, DataType, Int32Type, Int64Type}; -use datafusion_common::types::logical_string; +use datafusion_common::types::{logical_null, logical_string, NativeType}; use datafusion_common::{exec_err, internal_err, Result}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass, @@ -64,8 +64,16 @@ impl StrposFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), - Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Native(logical_null())], + NativeType::String, + ), ], Volatility::Immutable, ), diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 496f24abf6ed..70dd68590d9a 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -736,8 +736,11 @@ select specific_name, data_type, ordinal_position, parameter_mode, rid from info repeat Int64 2 IN 0 repeat Int64 2 IN 1 repeat Int64 2 IN 2 +repeat Int64 2 IN 3 repeat LargeUtf8 1 IN 1 repeat LargeUtf8 1 OUT 1 +repeat Null 1 IN 3 +repeat Null 1 OUT 3 repeat Utf8 1 IN 0 repeat Utf8 1 OUT 0 repeat Utf8 1 OUT 2 diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index e206aa16b8a9..93cda3a735f9 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -126,15 +126,15 @@ statement error SELECT abs(1, 2); # abs: unsupported argument type -query error DataFusion error: Error during planning: Function 'abs' expects NativeType::Numeric but received NativeType::String +query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Numeric but received NativeType::String, DataType: Utf8 SELECT abs('foo'); # abs: numeric string # TODO: In Postgres, '-1.2' is unknown type and interpreted to float8 so they don't fail on this query -query error DataFusion error: Error during planning: Function 'abs' expects NativeType::Numeric but received NativeType::String +query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Numeric but received NativeType::String, DataType: Utf8 select abs('-1.2'); -query error DataFusion error: Error during planning: Function 'abs' expects NativeType::Numeric but received NativeType::String +query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Numeric but received NativeType::String, DataType: Utf8 select abs(arrow_cast('-1.2', 'Utf8')); statement ok diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index aa14faf984e4..ac1853fb9a66 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -1568,13 +1568,13 @@ WHERE CAST(ROUND(b) as INT) = a ORDER BY CAST(ROUND(b) as INT); ---- logical_plan -01)Sort: CAST(round(CAST(annotated_data_finite2.b AS Float64)) AS Int32) ASC NULLS LAST -02)--Filter: CAST(round(CAST(annotated_data_finite2.b AS Float64)) AS Int32) = annotated_data_finite2.a -03)----TableScan: annotated_data_finite2 projection=[a0, a, b, c, d], partial_filters=[CAST(round(CAST(annotated_data_finite2.b AS Float64)) AS Int32) = annotated_data_finite2.a] +01)Sort: CAST(round(annotated_data_finite2.b) AS Int32) ASC NULLS LAST +02)--Filter: CAST(round(annotated_data_finite2.b) AS Int32) = annotated_data_finite2.a +03)----TableScan: annotated_data_finite2 projection=[a0, a, b, c, d], partial_filters=[CAST(round(annotated_data_finite2.b) AS Int32) = annotated_data_finite2.a] physical_plan -01)SortPreservingMergeExec: [CAST(round(CAST(b@2 AS Float64)) AS Int32) ASC NULLS LAST] +01)SortPreservingMergeExec: [CAST(round(b@2) AS Int32) ASC NULLS LAST] 02)--CoalesceBatchesExec: target_batch_size=8192 -03)----FilterExec: CAST(round(CAST(b@2 AS Float64)) AS Int32) = a@1 +03)----FilterExec: CAST(round(b@2) AS Int32) = a@1 04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a0, a, b, c, d], output_ordering=[a@1 ASC NULLS LAST, b@2 ASC NULLS LAST, c@3 ASC NULLS LAST], file_type=csv, has_header=true