devsisters · ghostdogpr · Feb 4, 2025 · Nov 26, 2025 · Nov 27, 2025
diff --git a/build.sbt b/build.sbt
@@ -4,28 +4,18 @@ ThisBuild / scalacOptions ++= Seq("-deprecation", "-release:8")
 
 ThisBuild / javacOptions ++= List("-target", "8", "-source", "8")
 
+ThisBuild / version := "0.0.1"
+
 Global / concurrentRestrictions := Seq(
   Tags.limit(Tags.Test, 1)
 )
 
-val Scala212 = "2.12.20"
-
-val Scala213 = "2.13.16"
-
-lazy val Spark35 = Spark("3.5.3")
-
-lazy val Spark34 = Spark("3.4.4")
-
-lazy val Spark33 = Spark("3.3.4")
+val Scala3 = "3.6.3"
 
 lazy val Spark32 = Spark("3.2.3")
 
-lazy val Spark31 = Spark("3.1.3")
-
 lazy val ScalaPB0_11 = ScalaPB("0.11.17")
 
-lazy val ScalaPB0_10 = ScalaPB("0.10.11")
-
 lazy val framelessDatasetName = settingKey[String]("frameless-dataset-name")
 
 lazy val framelessDatasetVersion = settingKey[String]("frameless-dataset-version")
@@ -50,14 +40,11 @@ lazy val `sparksql-scalapb` = (projectMatrix in file("sparksql-scalapb"))
   .defaultAxes()
   .settings(
     libraryDependencies ++= Seq(
-      "org.typelevel" %% framelessDatasetName.value % framelessDatasetVersion.value,
+      "org.typelevel" %% framelessDatasetName.value % framelessDatasetVersion.value cross CrossVersion.for3Use2_13,
       "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.value.scalapbVersion,
       "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.value.scalapbVersion % "protobuf",
-      "org.apache.spark" %% "spark-sql" % spark.value.sparkVersion % "provided",
-      "org.apache.spark" %% "spark-sql" % spark.value.sparkVersion % "test",
-      "org.scalatest" %% "scalatest" % "3.2.19" % "test",
-      "org.scalatestplus" %% "scalacheck-1-17" % "3.2.18.0" % "test",
-      "com.github.alexarchambault" %% "scalacheck-shapeless_1.16" % "1.3.1" % "test"
+      "org.apache.spark" %% "spark-sql" % spark.value.sparkVersion % "provided" cross CrossVersion.for3Use2_13,
+      "org.apache.spark" %% "spark-sql" % spark.value.sparkVersion % "test" cross CrossVersion.for3Use2_13
     ),
     spark := {
       virtualAxes.value
@@ -83,18 +70,18 @@ lazy val `sparksql-scalapb` = (projectMatrix in file("sparksql-scalapb"))
     },
     framelessDatasetName := {
       spark.value match {
-        case Spark35 | Spark34 | Spark33 => "frameless-dataset"
-        case Spark32                     => "frameless-dataset-spark32"
-        case Spark31                     => "frameless-dataset-spark31"
-        case _                           => ???
+//        case Spark35 | Spark34 | Spark33 => "frameless-dataset"
+        case Spark32 => "frameless-dataset-spark32"
+//        case Spark31                     => "frameless-dataset-spark31"
+        case _ => ???
       }
     },
     framelessDatasetVersion := {
       spark.value match {
-        case Spark35 | Spark34 | Spark33 => "0.16.0" // NPE in 3.4, 3.5 if older lib versions used
-        case Spark32                     => "0.15.0" // Spark3.2 support dropped in ver > 0.15.0
-        case Spark31                     => "0.14.0" // Spark3.1 support dropped in ver > 0.14.0
-        case _                           => ???
+//        case Spark35 | Spark34 | Spark33 => "0.16.0" // NPE in 3.4, 3.5 if older lib versions used
+        case Spark32 => "0.15.0" // Spark3.2 support dropped in ver > 0.15.0
+//        case Spark31                     => "0.14.0" // Spark3.1 support dropped in ver > 0.14.0
+        case _ => ???
       }
     },
     name := s"sparksql${spark.value.majorVersion}${spark.value.minorVersion}-${scalapb.value.idSuffix}",
@@ -107,46 +94,46 @@ lazy val `sparksql-scalapb` = (projectMatrix in file("sparksql-scalapb"))
     Test / run / fork := true,
     Test / javaOptions ++= Seq("-Xmx2G")
   )
+//  .customRow(
+//    scalaVersions = Seq(Scala212, Scala213),
+//    axisValues = Seq(Spark35, ScalaPB0_11, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
+//  .customRow(
+//    scalaVersions = Seq(Scala212, Scala213),
+//    axisValues = Seq(Spark34, ScalaPB0_11, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
+//  .customRow(
+//    scalaVersions = Seq(Scala212, Scala213),
+//    axisValues = Seq(Spark33, ScalaPB0_11, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
   .customRow(
-    scalaVersions = Seq(Scala212, Scala213),
-    axisValues = Seq(Spark35, ScalaPB0_11, VirtualAxis.jvm),
-    settings = Seq()
-  )
-  .customRow(
-    scalaVersions = Seq(Scala212, Scala213),
-    axisValues = Seq(Spark34, ScalaPB0_11, VirtualAxis.jvm),
-    settings = Seq()
-  )
-  .customRow(
-    scalaVersions = Seq(Scala212, Scala213),
-    axisValues = Seq(Spark33, ScalaPB0_11, VirtualAxis.jvm),
-    settings = Seq()
-  )
-  .customRow(
-    scalaVersions = Seq(Scala212, Scala213),
+    scalaVersions = Seq(Scala3),
     axisValues = Seq(Spark32, ScalaPB0_11, VirtualAxis.jvm),
     settings = Seq()
   )
-  .customRow(
-    scalaVersions = Seq(Scala212),
-    axisValues = Seq(Spark31, ScalaPB0_11, VirtualAxis.jvm),
-    settings = Seq()
-  )
-  .customRow(
-    scalaVersions = Seq(Scala212, Scala213),
-    axisValues = Seq(Spark33, ScalaPB0_10, VirtualAxis.jvm),
-    settings = Seq()
-  )
-  .customRow(
-    scalaVersions = Seq(Scala212, Scala213),
-    axisValues = Seq(Spark32, ScalaPB0_10, VirtualAxis.jvm),
-    settings = Seq()
-  )
-  .customRow(
-    scalaVersions = Seq(Scala212),
-    axisValues = Seq(Spark31, ScalaPB0_10, VirtualAxis.jvm),
-    settings = Seq()
-  )
+//  .customRow(
+//    scalaVersions = Seq(Scala212),
+//    axisValues = Seq(Spark31, ScalaPB0_11, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
+//  .customRow(
+//    scalaVersions = Seq(Scala212, Scala213),
+//    axisValues = Seq(Spark33, ScalaPB0_10, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
+//  .customRow(
+//    scalaVersions = Seq(Scala212, Scala213),
+//    axisValues = Seq(Spark32, ScalaPB0_10, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
+//  .customRow(
+//    scalaVersions = Seq(Scala212),
+//    axisValues = Seq(Spark31, ScalaPB0_10, VirtualAxis.jvm),
+//    settings = Seq()
+//  )
 
 ThisBuild / publishTo := sonatypePublishToBundle.value
 

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/CustomTypedEncoders.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/CustomTypedEncoders.scala
@@ -12,8 +12,8 @@ object CustomTypedEncoders {
   val timestampToSqlTimestamp: TypedEncoder[Timestamp] =
     fromInjection(
       Injection[Timestamp, SQLTimestamp](
-        { ts: Timestamp => SQLTimestamp(TimestampHelpers.toMicros(ts)) },
-        { timestamp: SQLTimestamp => TimestampHelpers.fromMicros(timestamp.us) }
+        { (ts: Timestamp) => SQLTimestamp(TimestampHelpers.toMicros(ts)) },
+        { (timestamp: SQLTimestamp) => TimestampHelpers.fromMicros(timestamp.us) }
       )
     )
 

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/FromCatalystHelpers.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/FromCatalystHelpers.scala
@@ -24,7 +24,7 @@ trait FromCatalystHelpers {
   def schemaOptions: SchemaOptions = protoSql.schemaOptions
 
   def pmessageFromCatalyst(
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       input: Expression
   ): Expression = {
     schemaOptions.messageEncoders.get(cmp.scalaDescriptor) match {
@@ -55,7 +55,7 @@ trait FromCatalystHelpers {
 
   def pmessageFromCatalyst(
       input: Expression,
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       args: Seq[Expression]
   ): Expression = {
     val outputType = ObjectType(classOf[PValue])
@@ -74,7 +74,7 @@ trait FromCatalystHelpers {
   }
 
   def fieldFromCatalyst(
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       fd: FieldDescriptor,
       input: Expression
   ): Expression = {
@@ -118,7 +118,7 @@ trait FromCatalystHelpers {
   }
 
   def singleFieldValueFromCatalyst(
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       fd: FieldDescriptor,
       input: Expression
   ): Expression = {
@@ -182,7 +182,7 @@ case class MyUnresolvedCatalystToExternalMap(
     @transient keyFunction: Expression => Expression,
     @transient valueFunction: Expression => Expression,
     mapType: MapType,
-    collClass: Class[_]
+    collClass: Class[?]
 )
 
 object MyCatalystToExternalMap {

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/JavaHelpers.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/JavaHelpers.scala
@@ -7,7 +7,7 @@ import scalapb.{GeneratedEnum, GeneratedEnumCompanion, GeneratedMessage, Generat
 
 object JavaHelpers {
   def enumToString(
-      cmp: GeneratedEnumCompanion[_],
+      cmp: GeneratedEnumCompanion[?],
       value: GeneratedEnum
   ): UTF8String = {
     UTF8String.fromString(
@@ -52,7 +52,7 @@ object JavaHelpers {
   }
 
   def penumFromString(
-      cmp: GeneratedEnumCompanion[_],
+      cmp: GeneratedEnumCompanion[?],
       inputUtf8: UTF8String
   ): PValue = {
     val input = inputUtf8.toString
@@ -73,7 +73,7 @@ object JavaHelpers {
     }
   }
 
-  def mkMap(cmp: GeneratedMessageCompanion[_], args: ArrayData): Map[FieldDescriptor, PValue] = {
+  def mkMap(cmp: GeneratedMessageCompanion[?], args: ArrayData): Map[FieldDescriptor, PValue] = {
     cmp.scalaDescriptor.fields
       .zip(args.array)
       .filter {
@@ -89,7 +89,7 @@ object JavaHelpers {
   }
 
   def mkPRepeatedMap(
-      mapEntryCmp: GeneratedMessageCompanion[_],
+      mapEntryCmp: GeneratedMessageCompanion[?],
       args: Vector[(PValue, PValue)]
   ): PValue = {
     val keyDesc = mapEntryCmp.scalaDescriptor.findFieldByNumber(1).get
@@ -102,7 +102,7 @@ object JavaHelpers {
   // ExternalMapToCatalyst only needs iterator. We create this view into s to
   // avoid making a copy.
   def mkMap(s: Seq[GeneratedMessage]): Map[Any, Any] =
-    scalapb.spark.internal.MapHelpers.fromIterator {
+    MapHelpers.fromIterator {
       if (s.isEmpty) Iterator.empty
       else {
         val cmp = s.head.companion

diff --git a/...3/scalapb/spark/internal/MapHelpers.scala → ...main/scala/scalapb/spark/MapHelpers.scala b/...3/scalapb/spark/internal/MapHelpers.scala → ...main/scala/scalapb/spark/MapHelpers.scala
@@ -1,4 +1,4 @@
-package scalapb.spark.internal
+package scalapb.spark
 
 private[spark] object MapHelpers {
   def fromIterator[K, V](it: => Iterator[(K, V)]): Map[K, V] = new Map[K, V] {

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/SchemaOptions.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/SchemaOptions.scala
@@ -8,15 +8,15 @@ import frameless.TypedEncoder
 case class SchemaOptions(
     columnNaming: ColumnNaming,
     retainPrimitiveWrappers: Boolean,
-    messageEncoders: Map[Descriptor, TypedEncoder[_]]
+    messageEncoders: Map[Descriptor, TypedEncoder[?]]
 ) {
   def withScalaNames = copy(columnNaming = ColumnNaming.ScalaNames)
 
   def withProtoNames = copy(columnNaming = ColumnNaming.ProtoNames)
 
   def withRetainedPrimitiveWrappers = copy(retainPrimitiveWrappers = true)
 
-  def withMessageEncoders(messageEncoders: Map[Descriptor, TypedEncoder[_]]) =
+  def withMessageEncoders(messageEncoders: Map[Descriptor, TypedEncoder[?]]) =
     copy(messageEncoders = messageEncoders)
 
   def addMessageEncoder[T <: GeneratedMessage](

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/ToCatalystHelpers.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/ToCatalystHelpers.scala
@@ -20,7 +20,7 @@ trait ToCatalystHelpers {
   def schemaOptions: SchemaOptions
 
   def messageToCatalyst(
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       input: Expression
   ): Expression = {
     schemaOptions.messageEncoders.get(cmp.scalaDescriptor) match {
@@ -52,7 +52,7 @@ trait ToCatalystHelpers {
   }
 
   def fieldGetterAndTransformer(
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       fd: FieldDescriptor
   ): (Expression => Expression, Expression => Expression) = {
     def messageFieldCompanion = cmp.messageCompanionForFieldNumber(fd.number)
@@ -73,7 +73,7 @@ trait ToCatalystHelpers {
               Nil
             ),
             "findFieldByNumber",
-            ObjectType(classOf[Option[_]]),
+            ObjectType(classOf[Option[?]]),
             Literal(fd.number) :: Nil
           ),
           "get",
@@ -86,26 +86,26 @@ trait ToCatalystHelpers {
         inputObject,
         "getFieldByNumber",
         if (fd.isRepeated)
-          ObjectType(classOf[Seq[_]])
+          ObjectType(classOf[Seq[?]])
         else
           ObjectType(messageFieldCompanion.defaultInstance.getClass),
         Literal(fd.number, IntegerType) :: Nil
       )
 
     if (!isMessage) {
-      (getField, { e: Expression => singularFieldToCatalyst(fd, e) })
+      (getField, { (e: Expression) => singularFieldToCatalyst(fd, e) })
     } else {
       (
         getFieldByNumber,
-        { e: Expression =>
+        { (e: Expression) =>
           messageToCatalyst(messageFieldCompanion, e)
         }
       )
     }
   }
 
   def fieldToCatalyst(
-      cmp: GeneratedMessageCompanion[_],
+      cmp: GeneratedMessageCompanion[?],
       fd: FieldDescriptor,
       inputObject: Expression
   ): Expression = {
@@ -132,7 +132,7 @@ trait ToCatalystHelpers {
         ExternalMapToCatalyst(
           StaticInvoke(
             JavaHelpers.getClass,
-            ObjectType(classOf[Map[_, _]]),
+            ObjectType(classOf[Map[?, ?]]),
             "mkMap",
             fieldGetter(inputObject) :: Nil
           ),
@@ -152,7 +152,7 @@ trait ToCatalystHelpers {
       else {
         val getter = StaticInvoke(
           JavaHelpers.getClass,
-          ObjectType(classOf[Vector[_]]),
+          ObjectType(classOf[Vector[?]]),
           "vectorFromPValue",
           fieldGetter(inputObject) :: Nil
         )

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/TypeMappers.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/TypeMappers.scala
@@ -9,9 +9,9 @@ import java.time.Instant
 object TypeMappers {
 
   implicit val googleTsToSqlTsMapper: TypeMapper[GoogleTimestamp, SQLTimestamp] =
-    TypeMapper({ googleTs: GoogleTimestamp =>
+    TypeMapper({ (googleTs: GoogleTimestamp) =>
       SQLTimestamp.from(Instant.ofEpochSecond(googleTs.seconds, googleTs.nanos))
-    })({ sqlTs: SQLTimestamp =>
+    })({ (sqlTs: SQLTimestamp) =>
       val instant = sqlTs.toInstant
       GoogleTimestamp(instant.getEpochSecond, instant.getNano)
     })

diff --git a/sparksql-scalapb/src/main/scala/scalapb/spark/TypedEncoders.scala b/sparksql-scalapb/src/main/scala/scalapb/spark/TypedEncoders.scala
@@ -28,11 +28,11 @@ trait TypedEncoders extends FromCatalystHelpers with ToCatalystHelpers with Seri
       val reads = Invoke(
         Literal.fromObject(cmp),
         "messageReads",
-        ObjectType(classOf[Reads[_]]),
+        ObjectType(classOf[Reads[?]]),
         Nil
       )
 
-      val read = Invoke(reads, "read", ObjectType(classOf[Function[_, _]]))
+      val read = Invoke(reads, "read", ObjectType(classOf[Function[?, ?]]))
 
       val ret = Invoke(read, "apply", ObjectType(ct.runtimeClass), expr :: Nil)
       ret