WIP Release infra stuff

NEXT_RELEASE_NOTES.md

@@ -0,0 +1,22 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+
+
+Content to be copied into the next release notes, after the Contributors section.
+This file is maintained manually!
+-->

build-logic/src/main/kotlin/publishing/PublishingHelperExtension.kt

@@ -49,12 +49,6 @@ constructor(objectFactory: ObjectFactory, project: Project) {
     objectFactory
       .fileProperty()
       .convention(project.provider { distributionDir.get().file("${baseName.get()}.tar.gz") })
-  val sourceTarballDigest =
-    objectFactory
-      .fileProperty()
-      .convention(
-        project.provider { distributionDir.get().file("${baseName.get()}.tar.gz.sha512") }
-      )
 
   val mailingLists = objectFactory.listProperty(String::class.java).convention(emptyList())
 

build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt

@@ -47,15 +47,8 @@ import org.gradle.plugins.signing.SigningPlugin
  * The task `sourceTarball` (available on the root project) generates a source tarball using `git
  * archive`.
  *
- * The task `releaseEmailTemplate` generates the release-vote email subject + body. Outputs on the
- * console and in the `build/distributions/` directory.
- *
  * Signing tip: If you want to use `gpg-agent`, set the `useGpgAgent` Gradle project property
  *
- * The following command publishes the project artifacts to your local maven repository, generates
- * the source tarball - and uses `gpg-agent` to sign all artifacts and the tarball. Note that this
- * requires a Git tag!
- *
  * ```
  * ./gradlew publishToMavenLocal sourceTarball -Prelease -PuseGpgAgent
  * ```

build-logic/src/main/kotlin/publishing/digest-task.kt

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package publishing
+
+import java.security.MessageDigest
+import javax.inject.Inject
+import org.gradle.api.DefaultTask
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+import org.gradle.work.DisableCachingByDefault
+
+@DisableCachingByDefault
+abstract class GenerateDigest @Inject constructor(objectFactory: ObjectFactory) : DefaultTask() {
+
+  @get:InputFile val file = objectFactory.fileProperty()
+  @get:Input val algorithm = objectFactory.property(String::class.java).convention("SHA-512")
+  @get:OutputFile
+  val outputFile =
+    objectFactory.fileProperty().convention {
+      val input = file.get().asFile
+      val algo = algorithm.get()
+      input.parentFile.resolve("${input.name}.${algo.replace("-", "").lowercase()}")
+    }
+
+  @TaskAction
+  fun generate() {
+    val input = file.get().asFile
+    val digestFile = outputFile.get().asFile
+    val md = MessageDigest.getInstance(algorithm.get())
+    input.inputStream().use {
+      val buffered = it.buffered(8192)
+      val buf = ByteArray(8192)
+      var rd: Int
+      while (true) {
+        rd = buffered.read(buf)
+        if (rd == -1) break
+        md.update(buf, 0, rd)
+      }
+
+      digestFile.writeText(
+        md.digest().joinToString(separator = "") { eachByte -> "%02x".format(eachByte) } +
+          "  ${input.name}"
+      )
+    }
+  }
+}

build-logic/src/main/kotlin/publishing/rootProject.kt

@@ -19,14 +19,11 @@
 
 package publishing
 
-import io.github.gradlenexus.publishplugin.NexusPublishExtension
 import io.github.gradlenexus.publishplugin.NexusPublishPlugin
-import io.github.gradlenexus.publishplugin.internal.StagingRepositoryDescriptorRegistryBuildService
 import org.gradle.api.Project
-import org.gradle.api.services.BuildServiceRegistration
+import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.Exec
 import org.gradle.kotlin.dsl.apply
-import org.gradle.kotlin.dsl.named
 import org.gradle.kotlin.dsl.register
 import org.gradle.plugins.signing.Sign
 
@@ -41,11 +38,22 @@ internal fun configureOnRootProject(project: Project) =
     val isRelease = project.hasProperty("release")
     val isSigning = isRelease || project.hasProperty("signArtifacts")
 
+    val cleanDistributionsDir = tasks.register<Delete>("cleanDistributionsDir")
+    cleanDistributionsDir.configure {
+      outputs.cacheIf { false }
+
+      val e = project.extensions.getByType(PublishingHelperExtension::class.java)
+      delete(e.distributionDir)
+    }
+
     val sourceTarball = tasks.register<Exec>("sourceTarball")
     sourceTarball.configure {
       group = "build"
       description =
         "Generate a source tarball for a release to be uploaded to dist.apache.org/repos/dist"
+      outputs.cacheIf { false }
+
+      dependsOn(cleanDistributionsDir)
 
       val e = project.extensions.getByType(PublishingHelperExtension::class.java)
       doFirst { mkdir(e.distributionDir) }
@@ -61,152 +69,28 @@ internal fun configureOnRootProject(project: Project) =
       workingDir(project.projectDir)
     }
 
-    val digestSourceTarball = tasks.register("digestSourceTarball")
-    digestSourceTarball.configure {
-      mustRunAfter(sourceTarball)
-
-      doFirst {
-        val e = project.extensions.getByType(PublishingHelperExtension::class.java)
-        generateDigest(e.sourceTarball.get().asFile, e.sourceTarballDigest.get().asFile, "SHA-512")
-      }
-    }
-
-    sourceTarball.configure { finalizedBy(digestSourceTarball) }
-
-    if (isSigning) {
-      val signSourceTarball = tasks.register<Sign>("signSourceTarball")
-      signSourceTarball.configure {
+    val digestSourceTarball =
+      tasks.register<GenerateDigest>("digestSourceTarball") {
+        description = "Generate the source tarball digest"
         mustRunAfter(sourceTarball)
-        doFirst {
+        file.set {
           val e = project.extensions.getByType(PublishingHelperExtension::class.java)
-          sign(e.sourceTarball.get().asFile)
+          e.sourceTarball.get().asFile
         }
       }
-      sourceTarball.configure { finalizedBy(signSourceTarball) }
-    }
-
-    val releaseEmailTemplate = tasks.register("releaseEmailTemplate")
-    releaseEmailTemplate.configure {
-      group = "publishing"
-      description =
-        "Generate release-vote email subject + body, including the staging repository URL, if run during the Maven release."
-
-      mustRunAfter("initializeApacheStagingRepository")
 
-      doFirst {
-        val e = project.extensions.getByType(PublishingHelperExtension::class.java)
-        val asfName = e.asfProjectName.get()
-
-        val gitInfo = MemoizedGitInfo.gitInfo(rootProject)
-        val gitCommitId = gitInfo["Apache-Polaris-Build-Git-Head"]
-
-        val repos = project.extensions.getByType(NexusPublishExtension::class.java).repositories
-        val repo = repos.iterator().next()
+    sourceTarball.configure { finalizedBy(digestSourceTarball) }
 
-        val stagingRepositoryUrlRegistryRegistration =
-          gradle.sharedServices.registrations.named<
-            BuildServiceRegistration<StagingRepositoryDescriptorRegistryBuildService, *>
-          >(
-            "stagingRepositoryUrlRegistry"
-          )
-        val staginRepoUrl =
-          if (stagingRepositoryUrlRegistryRegistration.isPresent) {
-            val stagingRepositoryUrlRegistryBuildServiceRegistration =
-              stagingRepositoryUrlRegistryRegistration.get()
-            val stagingRepositoryUrlRegistryService =
-              stagingRepositoryUrlRegistryBuildServiceRegistration.getService()
-            if (stagingRepositoryUrlRegistryService.isPresent) {
-              val registry = stagingRepositoryUrlRegistryService.get().registry
-              try {
-                val stagingRepoDesc = registry.get(repo.name)
-                val stagingRepoId = stagingRepoDesc.stagingRepositoryId
-                "https://repository.apache.org/content/repositories/$stagingRepoId/"
-              } catch (e: IllegalStateException) {
-                "NO STAGING REPOSITORY ($e)"
-              }
-            } else {
-              "NO STAGING REPOSITORY (no registry service) !!"
-            }
-          } else {
-            "NO STAGING REPOSITORY (no build service) !!"
+    if (isSigning) {
+      val signSourceTarball =
+        tasks.register<Sign>("signSourceTarball") {
+          description = "Sign the source tarball"
+          mustRunAfter(sourceTarball)
+          doFirst {
+            val e = project.extensions.getByType(PublishingHelperExtension::class.java)
+            sign(e.sourceTarball.get().asFile)
           }
-
-        val asfProjectName = fetchAsfProjectName(asfName)
-
-        val versionNoRc = version.toString().replace("-rc-?[0-9]+".toRegex(), "")
-
-        val subjectFile = e.distributionFile("vote-email-subject.txt").relativeTo(projectDir)
-        val bodyFile = e.distributionFile("vote-email-body.txt").relativeTo(projectDir)
-
-        val emailSubject = "[VOTE] Release $asfProjectName $version"
-        subjectFile.writeText(emailSubject)
-
-        val emailBody =
-          """
-              Hi everyone,
-
-              I propose that we release the following RC as the official
-              $asfProjectName $versionNoRc release.
-
-              * This corresponds to the tag: apache-$asfName-$version
-              * https://github.com/apache/$asfName/commits/apache-$asfName-$version
-              * https://github.com/apache/$asfName/tree/$gitCommitId
-
-              The release tarball, signature, and checksums are here:
-              * https://dist.apache.org/repos/dist/dev/incubator/$asfName/apache-$asfName-$version
-
-              You can find the KEYS file here:
-              * https://dist.apache.org/repos/dist/release/incubator/$asfName/KEYS
-
-              Convenience binary artifacts are staged on Nexus. The Maven repository URL is:
-              * $staginRepoUrl
-
-              Please download, verify, and test.
-
-              Please vote in the next 72 hours.
-
-              [ ] +1 Release this as Apache $asfName $version
-              [ ] +0
-              [ ] -1 Do not release this because...
-
-              Only PPMC members and mentors have binding votes, but other community members are
-              encouraged to cast non-binding votes. This vote will pass if there are
-              3 binding +1 votes and more binding +1 votes than -1 votes.
-
-              NB: if this vote pass, a new vote has to be started on the Incubator general mailing
-              list.
-
-              Thanks
-              Regards
-            """
-
-        logger.lifecycle(
-          """
-
-
-              The email for your release vote mail:
-              -------------------------------------
-
-              Suggested subject: (also in file $subjectFile) 
-
-              $emailSubject
-
-              Suggested body: (also in file $bodyFile)
-
-              $emailBody
-
-              """
-            .trimIndent()
-        )
-        bodyFile.writeText(emailBody.trimIndent())
-      }
-    }
-
-    if (isRelease) {
-      sourceTarball.configure { finalizedBy(releaseEmailTemplate) }
-    }
-
-    afterEvaluate {
-      tasks.named("closeApacheStagingRepository") { mustRunAfter(releaseEmailTemplate) }
+        }
+      sourceTarball.configure { finalizedBy(signSourceTarball) }
     }
   }

build-logic/src/main/kotlin/publishing/util.kt

@@ -23,10 +23,8 @@ import groovy.json.JsonException
 import groovy.json.JsonSlurper
 import groovy.util.Node
 import groovy.util.NodeList
-import java.io.File
 import java.io.FileNotFoundException
 import java.net.URI
-import java.security.MessageDigest
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.component.ModuleComponentSelector
 import org.gradle.api.artifacts.result.DependencyResult
@@ -58,25 +56,6 @@ internal fun xmlNode(node: Node?, child: String): Node? {
   return null
 }
 
-internal fun generateDigest(input: File, output: File, algorithm: String) {
-  val md = MessageDigest.getInstance(algorithm)
-  input.inputStream().use {
-    val buffered = it.buffered(8192)
-    val buf = ByteArray(8192)
-    var rd: Int
-    while (true) {
-      rd = buffered.read(buf)
-      if (rd == -1) break
-      md.update(buf, 0, rd)
-    }
-
-    output.writeText(
-      md.digest().joinToString(separator = "") { eachByte -> "%02x".format(eachByte) } +
-        "  ${input.name}"
-    )
-  }
-}
-
 internal fun <T : Any> unsafeCast(o: Any?): T {
   @Suppress("UNCHECKED_CAST")
   return o as T

build.gradle.kts

@@ -110,6 +110,9 @@ tasks.named<RatTask>("rat").configure {
   excludes.add("client/python/.openapi-generator/**")
   excludes.add("regtests/output/**")
 
+  excludes.add("releases/test/bats/**")
+  excludes.add("releases/test/test_helper/**")
+
   excludes.add("**/*.ipynb")
   excludes.add("**/*.iml")
   excludes.add("**/*.iws")