[SPARK-50807][BUILD] Upgrade Scala to 2.13.16

### What changes were proposed in this pull request?

This PR aims to Upgrade Scala to 2.13.16.

For now, this is a draft because the following plugins are not ready. So, we are using this PR to track the readiness.
- [x] lightbend/genjavadoc#375
- [x] com-lihaoyi/Ammonite#1597
    - scala/scala#10868 (comment)

### Why are the changes needed?

To deliver the latest Scala 2.13.x version to Apache Spark 4.0.0.
- https://github.com/scala/scala/releases/tag/v2.13.16

### Does this PR introduce _any_ user-facing change?

No because Apache Spark 4 is not released yet.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #49478 from dongjoon-hyun/SPARK-50807.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
(cherry picked from commit 585f434)
Signed-off-by: yangjie01 <[email protected]>

LICENSE-binary

@@ -296,7 +296,6 @@ jakarta.inject:jakarta.inject-api
 jakarta.validation:jakarta.validation-api
 javax.jdo:jdo-api
 joda-time:joda-time
-net.java.dev.jna:jna
 net.sf.opencsv:opencsv
 net.sf.supercsv:super-csv
 net.sf.jpam:jpam

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -120,7 +120,7 @@ jakarta.validation-api/3.0.2//jakarta.validation-api-3.0.2.jar
 jakarta.ws.rs-api/3.0.0//jakarta.ws.rs-api-3.0.0.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
 janino/3.1.9//janino-3.1.9.jar
-java-diff-utils/4.12//java-diff-utils-4.12.jar
+java-diff-utils/4.15//java-diff-utils-4.15.jar
 java-xmlbuilder/1.2//java-xmlbuilder-1.2.jar
 javassist/3.30.2-GA//javassist-3.30.2-GA.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
@@ -144,8 +144,7 @@ jjwt-api/0.12.6//jjwt-api-0.12.6.jar
 jjwt-impl/0.12.6//jjwt-impl-0.12.6.jar
 jjwt-jackson/0.12.6//jjwt-jackson-0.12.6.jar
 jline/2.14.6//jline-2.14.6.jar
-jline/3.26.3//jline-3.26.3.jar
-jna/5.14.0//jna-5.14.0.jar
+jline/3.27.1/jdk8/jline-3.27.1-jdk8.jar
 joda-time/2.13.0//joda-time-2.13.0.jar
 jodd-core/3.5.2//jodd-core-3.5.2.jar
 jpam/1.1//jpam-1.1.jar
@@ -253,11 +252,11 @@ py4j/0.10.9.9//py4j-0.10.9.9.jar
 remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
 rocksdbjni/9.8.4//rocksdbjni-9.8.4.jar
 scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar
-scala-compiler/2.13.15//scala-compiler-2.13.15.jar
-scala-library/2.13.15//scala-library-2.13.15.jar
+scala-compiler/2.13.16//scala-compiler-2.13.16.jar
+scala-library/2.13.16//scala-library-2.13.16.jar
 scala-parallel-collections_2.13/1.2.0//scala-parallel-collections_2.13-1.2.0.jar
 scala-parser-combinators_2.13/2.4.0//scala-parser-combinators_2.13-2.4.0.jar
-scala-reflect/2.13.15//scala-reflect-2.13.15.jar
+scala-reflect/2.13.16//scala-reflect-2.13.16.jar
 scala-xml_2.13/2.3.0//scala-xml_2.13-2.3.0.jar
 slf4j-api/2.0.16//slf4j-api-2.0.16.jar
 snakeyaml-engine/2.8//snakeyaml-engine-2.8.jar

docs/_config.yml

@@ -22,7 +22,7 @@ include:
 SPARK_VERSION: 4.0.0-SNAPSHOT
 SPARK_VERSION_SHORT: 4.0.0
 SCALA_BINARY_VERSION: "2.13"
-SCALA_VERSION: "2.13.15"
+SCALA_VERSION: "2.13.16"
 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK
 SPARK_GITHUB_URL: https://github.com/apache/spark
 # Before a new release, we should:

pom.xml

@@ -169,7 +169,7 @@
     <!-- managed up from 3.2.1 for SPARK-11652 -->
     <commons.collections.version>3.2.2</commons.collections.version>
     <commons.collections4.version>4.4</commons.collections4.version>
-    <scala.version>2.13.15</scala.version>
+    <scala.version>2.13.16</scala.version>
     <scala.binary.version>2.13</scala.binary.version>
     <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
     <scala-maven-plugin.version>4.9.2</scala-maven-plugin.version>
@@ -230,7 +230,7 @@
     and ./python/packaging/connect/setup.py too.
     -->
     <arrow.version>18.1.0</arrow.version>
-    <ammonite.version>3.0.0</ammonite.version>
+    <ammonite.version>3.0.1</ammonite.version>
     <jjwt.version>0.12.6</jjwt.version>
 
     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
@@ -2657,14 +2657,6 @@
                     <maxJdkVersion>${java.version}</maxJdkVersion>
                     <ignoredScopes>test</ignoredScopes>
                     <ignoredScopes>provided</ignoredScopes>
-                    <ignoreClasses>
-                      <!--
-                        The package `org.jline.terminal.impl.ffm.*` contains some class files
-                        that are not compatible with JDK17 (only JDK21 is supported).
-                        However, it will not cause problems for use. See: https://github.com/scala/bug/issues/12994
-                      -->
-                      <ignoreClass>org.jline.terminal.impl.ffm.*</ignoreClass>
-                    </ignoreClasses>
                   </enforceBytecodeVersion>
                 </rules>
               </configuration>