Skip to content

Commit b633ad3

Browse files
yaooqinndongjoon-hyun
authored andcommitted
[SPARK-53518][SQL] No truncation for catalogString of User Defined Type
### What changes were proposed in this pull request? `catalogString` of User Defined Type is mistakenly truncated, which leads to catalog errors. ### Why are the changes needed? bugfix ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? New Unit Test ### Was this patch authored or co-authored using generative AI tooling? no Closes #52263 from yaooqinn/SPARK-53518. Authored-by: Kent Yao <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 8575d09 commit b633ad3

File tree

3 files changed

+28
-1
lines changed

3 files changed

+28
-1
lines changed

sql/api/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
9393
case _ => false
9494
}
9595

96-
override def catalogString: String = sqlType.simpleString
96+
override def catalogString: String = sqlType.catalogString
9797

9898
/**
9999
* This method is used to convert the value of a UDT to a string representation.

sql/catalyst/src/test/scala/org/apache/spark/sql/types/TestUDT.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.spark.sql.types
1919

20+
import org.apache.spark.sql.Row
2021
import org.apache.spark.sql.catalyst.InternalRow
2122
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
2223
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
@@ -132,3 +133,22 @@ private[spark] class ExampleSubTypeUDT extends UserDefinedType[IExampleSubType]
132133

133134
override def userClass: Class[IExampleSubType] = classOf[IExampleSubType]
134135
}
136+
137+
138+
class ExampleIntRowUDT(cols: Int) extends UserDefinedType[Row] {
139+
override def sqlType: DataType = {
140+
StructType((0 until cols).map(i =>
141+
StructField(s"col$i", IntegerType, nullable = false)))
142+
}
143+
144+
override def serialize(obj: Row): InternalRow = {
145+
InternalRow.fromSeq(obj.toSeq)
146+
}
147+
148+
override def deserialize(datum: Any): Row = {
149+
val internalRow = datum.asInstanceOf[InternalRow]
150+
Row.fromSeq(internalRow.toSeq(sqlType.asInstanceOf[StructType]))
151+
}
152+
153+
override def userClass: Class[Row] = classOf[Row]
154+
}

sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,4 +330,11 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
330330
spark.range(10).map(i => Year.of(i.toInt + 2018)),
331331
(0 to 9).map(i => Year.of(i + 2018)): _*)
332332
}
333+
334+
test("SPARK-53518: No truncation for catalogString of User Defined Type") {
335+
withSQLConf(SQLConf.MAX_TO_STRING_FIELDS.key -> "3") {
336+
val string = new ExampleIntRowUDT(4).catalogString
337+
assert(string == "struct<col0:int,col1:int,col2:int,col3:int>")
338+
}
339+
}
333340
}

0 commit comments

Comments
 (0)