diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml index 1a5b0902703ab..af153ad9bcbab 100644 --- a/.github/actions/setup-gradle/action.yml +++ b/.github/actions/setup-gradle/action.yml @@ -42,7 +42,7 @@ runs: distribution: temurin java-version: ${{ inputs.java-version }} - name: Setup Gradle - uses: gradle/actions/setup-gradle@94baf225fe0a508e581a564467443d0e2379123b # v4.3.0 + uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3 env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true with: diff --git a/build.gradle b/build.gradle index 26ca93d0738e8..16a133929e7dd 100644 --- a/build.gradle +++ b/build.gradle @@ -29,22 +29,21 @@ buildscript { } plugins { - id 'com.github.ben-manes.versions' version '0.48.0' + id 'com.github.ben-manes.versions' version '0.52.0' id 'idea' id 'jacoco' id 'java-library' - id 'org.owasp.dependencycheck' version '8.2.1' + id 'org.owasp.dependencycheck' version '12.1.3' id 'org.nosphere.apache.rat' version "0.8.1" id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}" - id "com.github.spotbugs" version '6.2.3' apply false + id "com.github.spotbugs" version '6.2.5' apply false id 'org.scoverage' version '8.0.3' apply false - id 'com.gradleup.shadow' version '8.3.6' apply false - id 'com.diffplug.spotless' version "6.25.0" + id 'com.gradleup.shadow' version '9.0.2' apply false + id 'com.diffplug.spotless' version "7.2.1" } ext { - gradleVersion = versions.gradle minClientJavaVersion = 11 minNonClientJavaVersion = 17 modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams-scala", ":test-common:test-common-util"] @@ -275,6 +274,7 @@ if (repo != null) { // And some of the files that we have checked in should also be excluded from this check excludes.addAll([ '**/.git/**', + '**/.gitkeep', '**/build/**', '.github/pull_request_template.md', 'CONTRIBUTING.md', @@ -299,7 +299,7 @@ if (repo != null) { } else { rat.enabled = false } -println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}") +println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}") println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries") subprojects { @@ -330,6 +330,23 @@ subprojects { tasks.register('uploadArchives').configure { dependsOn(publish) } } + def submodulesToIncludeInDistribution = ['core', 'tools', 'trogdor', 'shell', 'api', 'runtime', 'transforms', + 'json', 'file', 'basic-auth-extension', 'mirror', 'mirror-client', + 'streams', 'streams-scala', 'test-utils', 'examples', 'tools-api'] + + def shouldModuleResolveRuntimeClasspath = (submodulesToIncludeInDistribution.contains(project.name)) + + if (shouldModuleResolveRuntimeClasspath) { + configurations.create('resolvedRuntimeClasspath') { + extendsFrom(configurations.runtimeClasspath) + canBeResolved = true + } + } + + tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false + } + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' @@ -1170,71 +1187,6 @@ project(':core') { standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream() } - task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs', - 'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs', - 'genKafkaConfigDocs', 'genTopicConfigDocs', 'genGroupConfigDocs', - ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs', - ':connect:runtime:genConnectPredicateDocs', - ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', - ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs', - ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs', - ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs', - ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs', - ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) { - archiveClassifier = 'site-docs' - compression = Compression.GZIP - from project.file("$rootDir/docs") - into 'site-docs' - duplicatesStrategy 'exclude' - } - - tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { - into "kafka_${versions.baseScala}-${archiveVersion.get()}" - compression = Compression.GZIP - from(project.file("$rootDir/bin")) { into "bin/" } - from(project.file("$rootDir/config")) { into "config/" } - from(project.file("$rootDir/licenses")) { into "licenses/" } - from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")} - from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")} - from(configurations.runtimeClasspath) { into("libs/") } - from(configurations.archives.artifacts.files) { into("libs/") } - from(configurations.releaseOnly) { into("libs/") } - from(project.siteDocsTar) { into("site-docs/") } - from(project(':tools').jar) { into("libs/") } - from(project(':tools').configurations.runtimeClasspath) { into("libs/") } - from(project(':trogdor').jar) { into("libs/") } - from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") } - from(project(':shell').jar) { into("libs/") } - from(project(':shell').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:api').jar) { into("libs/") } - from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:runtime').jar) { into("libs/") } - from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:transforms').jar) { into("libs/") } - from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:json').jar) { into("libs/") } - from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:file').jar) { into("libs/") } - from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:basic-auth-extension').jar) { into("libs/") } - from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:mirror').jar) { into("libs/") } - from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } - from(project(':connect:mirror-client').jar) { into("libs/") } - from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams').jar) { into("libs/") } - from(project(':streams').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams:streams-scala').jar) { into("libs/") } - from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams:test-utils').jar) { into("libs/") } - from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") } - from(project(':streams:examples').jar) { into("libs/") } - from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } - from(project(':tools:tools-api').jar) { into("libs/") } - from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") } - duplicatesStrategy 'exclude' - } - jar { dependsOn copyDependantLibs } @@ -3904,6 +3856,109 @@ project(':connect:test-plugins') { } } +project(':distribution') { + base { + archivesName = "kafka_${versions.baseScala}" + } + + configurations { + includeIntoDistribution + } + + dependencies { + includeIntoDistribution project(path: ':core', configuration: 'releaseOnly') + includeIntoDistribution project(path: ':core', configuration: 'archives') + includeIntoDistribution project(path: ':core', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':tools', configuration: 'archives') + includeIntoDistribution project(path: ':tools', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':trogdor', configuration: 'archives') + includeIntoDistribution project(path: ':trogdor', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':shell', configuration: 'archives') + includeIntoDistribution project(path: ':shell', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:api', configuration: 'archives') + includeIntoDistribution project(path: ':connect:api', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:runtime', configuration: 'archives') + includeIntoDistribution project(path: ':connect:runtime', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:transforms', configuration: 'archives') + includeIntoDistribution project(path: ':connect:transforms', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:json', configuration: 'archives') + includeIntoDistribution project(path: ':connect:json', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:file', configuration: 'archives') + includeIntoDistribution project(path: ':connect:file', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'archives') + includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:mirror', configuration: 'archives') + includeIntoDistribution project(path: ':connect:mirror', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'archives') + includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams', configuration: 'archives') + includeIntoDistribution project(path: ':streams', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'archives') + includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams:test-utils', configuration: 'archives') + includeIntoDistribution project(path: ':streams:test-utils', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':streams:examples', configuration: 'archives') + includeIntoDistribution project(path: ':streams:examples', configuration: 'resolvedRuntimeClasspath') + includeIntoDistribution project(path: ':tools:tools-api', configuration: 'archives') + includeIntoDistribution project(path: ':tools:tools-api', configuration: 'resolvedRuntimeClasspath') + } + + task siteDocsTar(dependsOn: [':core:genProtocolErrorDocs', ':core:genProtocolTypesDocs', ':core:genProtocolApiKeyDocs', ':core:genProtocolMessageDocs', + ':core:genAdminClientConfigDocs', ':core:genProducerConfigDocs', ':core:genConsumerConfigDocs', + ':core:genKafkaConfigDocs', ':core:genTopicConfigDocs', ':core:genGroupConfigDocs', + ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs', + ':connect:runtime:genConnectPredicateDocs', + ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', + ':streams:genStreamsConfigDocs', ':core:genConsumerMetricsDocs', ':core:genProducerMetricsDocs', + ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs', + ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs', + ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs', + ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) { + archiveClassifier = 'site-docs' + compression = Compression.GZIP + from project.file("$rootDir/docs") + into 'site-docs' + destinationDirectory = project(':core').layout.buildDirectory.dir('distributions') + duplicatesStrategy 'exclude' + preserveFileTimestamps = true + } + + tasks.create(name: "releaseTarGz", dependsOn: project(':core').configurations.archives.artifacts, type: Tar) { + into "kafka_${versions.baseScala}-${archiveVersion.get()}" + compression = Compression.GZIP + from(project.file("$rootDir/bin")) { into "bin/" } + from(project.file("$rootDir/config")) { into "config/" } + from(project.file("$rootDir/licenses")) { into "licenses/" } + from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")} + from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")} + from(project.siteDocsTar) { into("site-docs/") } + from(configurations.includeIntoDistribution) { into("libs/") } + destinationDirectory = project(':core').layout.buildDirectory.dir('distributions') + duplicatesStrategy 'exclude' + preserveFileTimestamps = true + filePermissions { + user { + read = true + write = true + } + group.read = true + other.read = true + } + dirPermissions { + unix('rwxr-xr-x') + } + eachFile { + if (name.endsWith( ".sh")) { + permissions { + user.execute = true + group.execute = true + other.execute = true + } + } + } + } +} + task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) { def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled } source = projectsWithJavadoc.collect { it.sourceSets.main.allJava } diff --git a/distribution/.gitkeep b/distribution/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 390c7e830c80d..fbcec1730e6fb 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -61,8 +61,8 @@ versions += [ commonsLang: "3.18.0", commonsValidator: "1.9.0", classgraph: "4.8.179", - gradle: "8.14.3", - grgit: "4.1.1", + gradle: "9.0.0", + grgit: "5.3.0", httpclient: "4.5.14", jackson: "2.19.0", jacoco: "0.8.13", @@ -126,7 +126,7 @@ versions += [ snappy: "1.1.10.7", spotbugs: "4.9.4", mockOAuth2Server: "2.2.1", - zinc: "1.9.2", + zinc: "1.10.8", // When updating the zstd version, please do as well in docker/native/native-image-configs/resource-config.json // Also make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid zstd: "1.5.6-10", diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 78cb6e16a49f9..3e781fbad9c7f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +distributionSha256Sum=8fad3d78296ca518113f3d29016617c7f9367dc005f932bd9d93bf45ba46072b +distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index f4bb3360e17ee..795b507ea2329 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,7 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -112,20 +114,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac - -# Loop in case we encounter an error. -for attempt in 1 2 3; do - if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then - if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.3/gradle/wrapper/gradle-wrapper.jar"; then - rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" - # Pause for a bit before looping in case the server throttled us. - sleep 5 - continue - fi - fi -done - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -212,11 +201,24 @@ if "$cygwin" || "$msys" ; then fi + +# Loop in case we encounter an error. +for attempt in 1 2 3; do + if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then + if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.0.0/gradle/wrapper/gradle-wrapper.jar"; then + rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" + # Pause for a bit before looping in case the server throttled us. + sleep 5 + continue + fi + fi +done + # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -224,7 +226,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/settings.gradle b/settings.gradle index 7c37a046838d8..2e273faabaed7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -76,6 +76,7 @@ include 'clients', 'connect:transforms', 'coordinator-common', 'core', + 'distribution', 'examples', 'generator', 'group-coordinator', diff --git a/wrapper.gradle b/wrapper.gradle index d64aeab4de942..4cd9fc65f9d45 100644 --- a/wrapper.gradle +++ b/wrapper.gradle @@ -23,7 +23,7 @@ // and not the version installed on the machine running the task. // Read more about the wrapper here: https://docs.gradle.org/current/userguide/gradle_wrapper.html wrapper { - gradleVersion = project.gradleVersion + gradleVersion = versions.gradle } // Custom task to inject support for downloading the gradle wrapper jar if it doesn't exist. @@ -35,14 +35,12 @@ task bootstrapWrapper() { def wrapperBasePath = "\$APP_HOME/gradle/wrapper" def wrapperJarPath = wrapperBasePath + "/gradle-wrapper.jar" - // Add a trailing zero to the version if needed. - def fullVersion = project.gradleVersion.count(".") == 1 ? "${project.gradleVersion}.0" : versions.gradle // Leverages the wrapper jar checked into the gradle project on github because the jar isn't // available elsewhere. Using raw.githubusercontent.com instead of github.com because // github.com servers deprecated TLSv1/TLSv1.1 support some time ago, so older versions // of curl (built against OpenSSL library that doesn't support TLSv1.2) would fail to // fetch the jar. - def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$fullVersion/gradle/wrapper" + def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$versions.gradle/gradle/wrapper" def wrapperJarUrl = wrapperBaseUrl + "/gradle-wrapper.jar" def bootstrapString = """ @@ -59,13 +57,15 @@ task bootstrapWrapper() { done """.stripIndent() + String putBootstrapStringAbove = "# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script." + def wrapperScript = wrapper.scriptFile def wrapperLines = wrapperScript.readLines() wrapperScript.withPrintWriter { out -> def bootstrapWritten = false wrapperLines.each { line -> // Print the wrapper bootstrap before the first usage of the wrapper jar. - if (!bootstrapWritten && line.contains("gradle-wrapper.jar")) { + if (!bootstrapWritten && line.contains(putBootstrapStringAbove)) { out.println(bootstrapString) bootstrapWritten = true }