diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index 3310ff94bdeb..497d82a1c876 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -2,11 +2,17 @@ package dotty.tools.sbtplugin import sbt.* import sbt.Keys.* +import sbt.io.Using import scala.jdk.CollectionConverters.* +import scala.collection.mutable import java.nio.file.Files +import java.nio.ByteBuffer import xsbti.VirtualFileRef import sbt.internal.inc.Stamper import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport.scalaJSVersion +import org.objectweb.asm.* + +import dotty.tools.tasty.TastyHeaderUnpickler object ScalaLibraryPlugin extends AutoPlugin { @@ -14,6 +20,19 @@ object ScalaLibraryPlugin extends AutoPlugin { private val scala2Version = "2.13.16" + /** Scala 2 pickle annotation descriptors that should be stripped from class files */ + private val Scala2PickleAnnotations = Set( + "Lscala/reflect/ScalaSignature;", + "Lscala/reflect/ScalaLongSignature;" + ) + + /** Scala 2 attribute names that should be stripped from class files */ + private val Scala2PickleAttributes = Set("ScalaSig", "ScalaInlineInfo") + + /** Check if an annotation descriptor is a Scala 2 pickle annotation */ + private def isScala2PickleAnnotation(descriptor: String): Boolean = + Scala2PickleAnnotations.contains(descriptor) + object autoImport { val keepSJSIR = settingKey[Boolean]("Should we patch .sjsir too?") } @@ -21,6 +40,15 @@ object ScalaLibraryPlugin extends AutoPlugin { import autoImport._ override def projectSettings = Seq ( + // Settings to validate that JARs don't contain Scala 2 pickle annotations and have valid TASTY attributes + Compile / packageBin := (Compile / packageBin) + .map{ jar => + validateNoScala2Pickles(jar) + validateTastyAttributes(jar) + validateScalaAttributes(jar) + jar + } + .value, (Compile / manipulateBytecode) := { val stream = streams.value val target = (Compile / classDirectory).value @@ -46,6 +74,8 @@ object ScalaLibraryPlugin extends AutoPlugin { } var stamps = analysis.stamps + val classDir = (Compile / classDirectory).value + // Patch the files that are in the list for { (files, reference) <- patches @@ -56,8 +86,7 @@ object ScalaLibraryPlugin extends AutoPlugin { dest = target / (id.toString) ref <- dest.relativeTo((LocalRootProject / baseDirectory).value) } { - // Copy the files to the classDirectory - IO.copyFile(file, dest) + patchFile(input = file, output = dest, classDirectory = classDir) // Update the timestamp in the analysis stamps = stamps.markProduct( VirtualFileRef.of(s"$${BASE}/$ref"), @@ -65,11 +94,11 @@ object ScalaLibraryPlugin extends AutoPlugin { } - val overwrittenBinaries = Files.walk((Compile / classDirectory).value.toPath()) + val overwrittenBinaries = Files.walk(classDir.toPath()) .iterator() .asScala .map(_.toFile) - .map(_.relativeTo((Compile / classDirectory).value).get) + .map(_.relativeTo(classDir).get) .toSet for ((files, reference) <- patches) { @@ -77,7 +106,11 @@ object ScalaLibraryPlugin extends AutoPlugin { // Copy all the specialized classes in the stdlib // no need to update any stamps as these classes exist nowhere in the analysis for (orig <- diff; dest <- orig.relativeTo(reference)) { - IO.copyFile(orig, ((Compile / classDirectory).value / dest.toString())) + patchFile( + input = orig, + output = classDir / dest.toString(), + classDirectory = classDir, + ) } } @@ -103,6 +136,187 @@ object ScalaLibraryPlugin extends AutoPlugin { } (Set(jar)), target) } + /** Remove Scala 2 Pickles from class file and optionally add TASTY attribute. + * Also ensures the Scala attribute is present for all Scala-compiled classes. + * + * @param bytes the class file bytecode + * @param tastyUUID optional 16-byte UUID from the corresponding .tasty file (only for primary class) + */ + private def patchClassFile(bytes: Array[Byte], tastyUUID: Option[Array[Byte]]): Array[Byte] = { + val reader = new ClassReader(bytes) + val writer = new ClassWriter(0) + // Remove Scala 2 pickles and Scala signatures + val visitor = new ClassVisitor(Opcodes.ASM9, writer) { + override def visitAttribute(attr: Attribute): Unit = { + val shouldRemove = Scala2PickleAttributes.contains(attr.`type`) + if (!shouldRemove) super.visitAttribute(attr) + } + + override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = + if (isScala2PickleAnnotation(desc)) null + else super.visitAnnotation(desc, visible) + } + reader.accept(visitor, 0) + // Only add TASTY attribute for the primary class (not for inner/nested classes) + tastyUUID + .map(new TastyAttribute(_)) + .foreach(writer.visitAttribute) + // Add Scala attribute if not present and this is a Scala-compiled class + def isJavaSourced = extractSourceFile(bytes).exists(_.endsWith(".java")) + if (!hasScalaAttribute(bytes) && !isJavaSourced) { + writer.visitAttribute(new ScalaAttribute) + } + writer.toByteArray + } + + /** Apply the patches to given input file and write the result to the output. + * For .class files, strips Scala 2 pickles and adds TASTY attribute only for primary classes. + * + * The TASTY attribute is only added to the "primary" class for each .tasty file: + * - Inner/nested classes (e.g., Outer$Inner.class) don't get TASTY attribute + * - Companion objects (Foo$.class when Foo.class exists) don't get TASTY attribute + * - Only the class whose name matches the .tasty file name gets the attribute + * - Java source files don't produce .tasty files, so they are skipped + * + * Additionally validates that if the original class file (before patching) had a TASTY + * attribute, the patched version will also have one. This prevents accidentally losing + * TASTY attributes during the patching process. + * + * @param input the input file (.class or .sjsir) + * @param output the output file location + * @param classDirectory the class directory to look for .tasty files + */ + def patchFile(input: File, output: File, classDirectory: File): File = { + if (input.getName.endsWith(".sjsir")) { + // For .sjsir files, we just copy the file + IO.copyFile(input, output) + return output + } + + // Extract the original TASTY UUID if the class file exists and has one + val originalTastyUUID: Option[Array[Byte]] = + if (output.exists()) extractTastyUUIDFromClass(IO.readBytes(output)) + else None + + val relativePath = output.relativeTo(classDirectory) + .getOrElse(sys.error(s"Patched file is not relative to class directory: $output")) + .getPath + val classPath = relativePath.stripSuffix(".class") + val basePath = classPath.split('$').head + + // Skip TASTY handling for Java-sourced classes (they don't have .tasty files) + val classfileBytes = IO.readBytes(input) + val isJavaSourced = extractSourceFile(classfileBytes).exists(_.endsWith(".java")) + val tastyUUID = + if (isJavaSourced) None + else { + val tastyFile = classDirectory / (basePath + ".tasty") + assert(tastyFile.exists(), s"TASTY file $tastyFile does not exist for $relativePath") + + // Only add TASTY attribute if this is the primary class (class path equals base path) + // Inner classes, companion objects ($), anonymous classes ($$anon), etc. don't get TASTY attribute + val isPrimaryClass = classPath == basePath + if (isPrimaryClass) Some(extractTastyUUID(IO.readBytes(tastyFile))) + else None + } + + // Validation to ensure that no new TASTY attributes are added or removed when compared with unpatched sources + (tastyUUID, originalTastyUUID) match { + case (None, None) => () // no TASTY attribute, no problem + case (Some(newUUID), Some(originalUUID)) => + assert(java.util.Arrays.equals(originalUUID, newUUID), + s"TASTY UUID mismatch for $relativePath: original=${originalUUID.map(b => f"$b%02x").mkString}, new=${newUUID.map(b => f"$b%02x").mkString}." + ) + case (Some(_), None) => sys.error(s"TASTY attribute defined, but not present in unpatched source $relativePath") + case (None, Some(_)) => sys.error(s"TASTY attribute missing, but present in unpatched $relativePath") + } + + IO.write(output, patchClassFile(classfileBytes, tastyUUID)) + output + } + + /** Check if class file bytecode contains Scala 2 pickle annotations or attributes */ + private def hasScala2Pickles(bytes: Array[Byte]): Boolean = { + var hasPickleAnnotation = false + var hasScalaSigAttr = false + var hasScalaInlineInfoAttr = false + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = { + if (isScala2PickleAnnotation(desc)) hasPickleAnnotation = true + null + } + override def visitAttribute(attr: Attribute): Unit = + if (Scala2PickleAttributes.contains(attr.`type`)) attr.`type` match { + case "ScalaSig" => hasScalaSigAttr = true + case "ScalaInlineInfo" => hasScalaInlineInfoAttr = true + } + } + new ClassReader(bytes).accept( + visitor, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES + ) + hasPickleAnnotation || hasScalaSigAttr || hasScalaInlineInfoAttr + } + + /** Check if class file bytecode contains a Scala attribute */ + private def hasScalaAttribute(bytes: Array[Byte]): Boolean = { + var hasScala = false + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitAttribute(attr: Attribute): Unit = { + if (attr.`type` == "Scala") hasScala = true + } + } + new ClassReader(bytes).accept( + visitor, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES + ) + hasScala + } + + /** Validate that all files produced by Scala compiler have a "Scala" attribute. + * Java-sourced files are excluded from this check since they don't have Scala attributes. + */ + def validateScalaAttributes(jar: File): Unit = { + val classFilesWithoutScala = Using.jarFile(verify = true)(jar) { jarFile => + jarFile + .entries().asScala + .filter(_.getName.endsWith(".class")) + .flatMap { entry => + Using.bufferedInputStream(jarFile.getInputStream(entry)) { inputStream => + val bytes = inputStream.readAllBytes() + // Skip Java-sourced files - they won't have Scala attributes + val isJavaSourced = extractSourceFile(bytes).exists(_.endsWith(".java")) + if (!isJavaSourced && !hasScalaAttribute(bytes)) Some(entry.getName) + else None + } + } + .toList + } + assert( + classFilesWithoutScala.isEmpty, + s"JAR ${jar.getName} contains ${classFilesWithoutScala.size} class files without 'Scala' attribute: ${classFilesWithoutScala.mkString("\n - ", "\n - ", "")}" + ) + } + + def validateNoScala2Pickles(jar: File): Unit = { + val classFilesWithPickles = Using.jarFile(verify = true)(jar){ jarFile => + jarFile + .entries().asScala + .filter(_.getName.endsWith(".class")) + .flatMap { entry => + Using.bufferedInputStream(jarFile.getInputStream(entry)){ inputStream => + if (hasScala2Pickles(inputStream.readAllBytes())) Some(entry.getName) + else None + } + } + .toList + } + assert( + classFilesWithPickles.isEmpty, + s"JAR ${jar.getName} contains ${classFilesWithPickles.size} class files with Scala 2 pickle annotations: ${classFilesWithPickles.mkString("\n - ", "\n - ", "")}" + ) + } + private lazy val filesToCopy = Set( "scala/Tuple1", "scala/Tuple2", @@ -144,4 +358,132 @@ object ScalaLibraryPlugin extends AutoPlugin { "scala/util/Sorting", ) + /** Extract the SourceFile attribute from class file bytecode */ + private def extractSourceFile(bytes: Array[Byte]): Option[String] = { + var sourceFile: Option[String] = None + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitSource(source: String, debug: String): Unit = + sourceFile = Option(source) + } + // Note: Don't use SKIP_DEBUG here - SourceFile is debug info and would be skipped + new ClassReader(bytes).accept( + visitor, + ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES + ) + sourceFile + } + + /** Extract the UUID bytes (16 bytes) from a TASTy file. + * + * Uses the official TastyHeaderUnpickler to parse the header and extract the UUID, + * ensuring correctness and validating the TASTy format. + */ + private def extractTastyUUID(tastyBytes: Array[Byte]): Array[Byte] = { + val unpickler = new TastyHeaderUnpickler(tastyBytes) + val header = unpickler.readFullHeader() + val uuid = header.uuid + + // Convert UUID (two longs) to 16-byte array in big-endian format + val buffer = ByteBuffer.allocate(16) + buffer.putLong(uuid.getMostSignificantBits) + buffer.putLong(uuid.getLeastSignificantBits) + buffer.array() + } + + /** Extract TASTY UUID from class file bytecode, if present */ + private def extractTastyUUIDFromClass(bytes: Array[Byte]): Option[Array[Byte]] = { + var result: Option[Array[Byte]] = None + val tastyPrototype = new Attribute("TASTY") { + override def read(cr: ClassReader, off: Int, len: Int, buf: Array[Char], codeOff: Int, labels: Array[Label]): Attribute = { + if (len == 16) { + val uuid = Array.tabulate[Byte](16)(i => cr.readByte(off + i).toByte) + result = Some(uuid) + } + this + } + } + new ClassReader(bytes).accept( + new ClassVisitor(Opcodes.ASM9) {}, + Array(tastyPrototype), + 0 + ) + result + } + + /** Validate TASTY attributes in the JAR: + * - If a .class file has a TASTY attribute, verify its UUID matches a .tasty file in the JAR + * - Every .tasty file must have at least one .class file with a matching TASTY attribute + * + * Note: .class files from Java sources don't have .tasty files and are allowed to not have TASTY attributes. + */ + def validateTastyAttributes(jar: File): Unit = { + Using.jarFile(verify = true)(jar) { jarFile => + // Build a map of .tasty file paths to their UUIDs + val tastyEntries = jarFile.entries().asScala + .filter(_.getName.endsWith(".tasty")) + .map { entry => + val bytes = Using.bufferedInputStream(jarFile.getInputStream(entry))(_.readAllBytes()) + val uuid = extractTastyUUID(bytes) + entry.getName -> uuid + } + .toMap + + val errors = mutable.ListBuffer.empty[String] + val referencedTastyFiles = mutable.Set.empty[String] + + // Check each .class file that has a TASTY attribute + jarFile.entries().asScala + .filter(e => e.getName.endsWith(".class")) + .foreach { entry => + val classBytes = Using.bufferedInputStream(jarFile.getInputStream(entry))(_.readAllBytes()) + val classPath = entry.getName + + // Only validate classes that have a TASTY attribute + extractTastyUUIDFromClass(classBytes).foreach[Unit] { classUUID => + // Find a .tasty file with matching UUID + tastyEntries.find{ case (path, tastyUUID) => + java.util.Arrays.equals(classUUID, tastyUUID) && { + val tastyName = file(path).getName().stripSuffix(".tasty") + val className = file(entry.getName()).getName().stripSuffix(".class") + // apparently 2 files might have the same UUID, e.g. param.scala and field.scala + className.startsWith(tastyName) + }} match { + case Some((path, _)) => + referencedTastyFiles += path + case None => + val uuidHex = classUUID.map(b => f"$b%02x").mkString + errors += s"$classPath: has TASTY attribute (UUID=$uuidHex) but no matching .tasty file found in JAR" + } + } + } + + // Check that every .tasty file has at least one .class file referencing it + val unreferencedTastyFiles = tastyEntries.keySet -- referencedTastyFiles + unreferencedTastyFiles.foreach { tastyPath => + errors += s"$tastyPath: no .class file with matching TASTY attribute found" + } + + assert( + errors.isEmpty, + s"JAR ${jar.getName} has ${errors.size} TASTY validation errors:\n - ${errors.mkString("\n - ")}" + ) + } + } + + /** Custom ASM Attribute for TASTY that can be written to class files */ + private class TastyAttribute(val uuid: Array[Byte]) extends Attribute("TASTY") { + override def write(classWriter: ClassWriter, code: Array[Byte], codeLength: Int, maxStack: Int, maxLocals: Int): ByteVector = { + val bv = new ByteVector(uuid.length) + bv.putByteArray(uuid, 0, uuid.length) + bv + } + } + + /** Custom ASM Attribute for Scala attribute marker (empty attribute) */ + private class ScalaAttribute extends Attribute("Scala") { + override def write(classWriter: ClassWriter, code: Array[Byte], codeLength: Int, maxStack: Int, maxLocals: Int): ByteVector = { + // Scala attribute is empty (length = 0x0) + new ByteVector(0) + } + } } diff --git a/project/build.sbt b/project/build.sbt index 9e96a2327deb..72216a873dbc 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -1,7 +1,16 @@ // Used by VersionUtil to get gitHash and commitDate libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit" % "4.11.0.201803080745-r" +libraryDependencies += "org.ow2.asm" % "asm" % "9.9" libraryDependencies += Dependencies.`jackson-databind` // Used for manipulating YAML files in sidebar generation script -libraryDependencies += "org.yaml" % "snakeyaml" % "2.4" \ No newline at end of file +libraryDependencies += "org.yaml" % "snakeyaml" % "2.4" + +Compile / unmanagedSourceDirectories ++= { + val root = baseDirectory.value.getParentFile() + Seq( + root / "tasty/src", + root / "tasty/src/dotty/tools/tasty/util", + ) +} diff --git a/project/stubs.scala b/project/stubs.scala new file mode 100644 index 000000000000..e9299a3e9388 --- /dev/null +++ b/project/stubs.scala @@ -0,0 +1,9 @@ +// Stubs for Scala 3 stdlib required to compile build unmanaged sources + +package scala { + package annotation { + package internal { + class sharable extends Annotation + } + } +} diff --git a/tasty/src/dotty/tools/tasty/TastyVersion.scala b/tasty/src/dotty/tools/tasty/TastyVersion.scala index b6474f7c7934..e4dd1ad91af3 100644 --- a/tasty/src/dotty/tools/tasty/TastyVersion.scala +++ b/tasty/src/dotty/tools/tasty/TastyVersion.scala @@ -20,7 +20,7 @@ case class TastyVersion private(major: Int, minor: Int, experimental: Int) { def validRange: String = { val min = TastyVersion(major, 0, 0) val max = if (experimental == 0) this else TastyVersion(major, minor - 1, 0) - val extra = Option.when(experimental > 0)(this) + val extra = Option(this).filter(_ => experimental > 0) s"stable TASTy from ${min.show} to ${max.show}${extra.fold("")(e => s", or exactly ${e.show}")}" } }