fix: Clean up multipart files from disk properly

.github/workflows/build.yml

@@ -49,15 +49,24 @@ jobs:
           java-version: "17"
           distribution: "zulu"
 
-      # TODO: Figure out how to simplify integration test run below
-
-      - name: Add Homebrew to PATH
-        run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
-
-      - name: Setup Container
+      # The integration tests seed secrets into a dev Vault container by invoking the
+      # `vault` CLI directly, so the CLI must be on PATH. Vault is no longer in
+      # homebrew-core, and `brew tap hashicorp/tap` eagerly validates every cask in the
+      # tap (a broken unrelated cask there fails the whole step), so we install a pinned,
+      # checksum-verified binary directly from releases.hashicorp.com instead.
+      - name: Install Vault CLI
+        env:
+          # Keep in lockstep with the dev Vault container image
+          # (service/src/test/resources/dependencies.yaml).
+          VAULT_VERSION: "1.14.4"
+          VAULT_SHA256: "2e94ba5f3e6b361847763a4c4fba87050221e76f02c3a118605ec56155a7a3cf"
         run: |-
-          brew tap hashicorp/tap
-          brew install hashicorp/tap/vault
+          set -euo pipefail
+          curl -fsSL -o vault.zip "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip"
+          echo "${VAULT_SHA256}  vault.zip" | sha256sum --check --status
+          sudo unzip -o vault.zip vault -d /usr/local/bin
+          rm vault.zip
+          vault --version
 
       - name: Gradle Build
         run: ./gradlew clean build -i --refresh-dependencies --parallel -x ktlint -x detekt

.github/workflows/release.yml

@@ -34,13 +34,24 @@ jobs:
           java-version: 17
           distribution: "zulu"
 
-      - name: Add Homebrew to PATH
-        run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
-
-      - name: Setup Vault Container
+      # The integration tests seed secrets into a dev Vault container by invoking the
+      # `vault` CLI directly, so the CLI must be on PATH. Vault is no longer in
+      # homebrew-core, and `brew tap hashicorp/tap` eagerly validates every cask in the
+      # tap (a broken unrelated cask there fails the whole step), so we install a pinned,
+      # checksum-verified binary directly from releases.hashicorp.com instead.
+      - name: Install Vault CLI
+        env:
+          # Keep in lockstep with the dev Vault container image
+          # (service/src/test/resources/dependencies.yaml).
+          VAULT_VERSION: "1.14.4"
+          VAULT_SHA256: "2e94ba5f3e6b361847763a4c4fba87050221e76f02c3a118605ec56155a7a3cf"
         run: |-
-          brew tap hashicorp/tap
-          brew install hashicorp/tap/vault
+          set -euo pipefail
+          curl -fsSL -o vault.zip "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip"
+          echo "${VAULT_SHA256}  vault.zip" | sha256sum --check --status
+          sudo unzip -o vault.zip vault -d /usr/local/bin
+          rm vault.zip
+          vault --version
 
       - name: Build with Gradle
         run: ./gradlew clean build --refresh-dependencies

service/src/main/kotlin/io/provenance/api/domain/usecase/objectStore/store/StoreFile.kt

@@ -13,7 +13,10 @@ import io.provenance.api.util.awaitAllBytes
 import io.provenance.api.util.buildLogMessage
 import io.provenance.entity.KeyType
 import io.provenance.scope.util.toUuid
+import kotlinx.coroutines.NonCancellable
+import kotlinx.coroutines.reactive.awaitFirstOrNull
 import kotlinx.coroutines.reactor.awaitSingle
+import kotlinx.coroutines.withContext
 import org.springframework.http.codec.multipart.FilePart
 import org.springframework.http.codec.multipart.FormFieldPart
 import org.springframework.http.codec.multipart.Part
@@ -28,7 +31,21 @@ class StoreFile(
     private val entityManager: EntityManager,
     private val storeObject: StoreObject,
 ) : AbstractUseCase<StoreFileRequestWrapper, StoreProtoResponse>() {
-    override suspend fun execute(args: StoreFileRequestWrapper): StoreProtoResponse {
+    override suspend fun execute(args: StoreFileRequestWrapper): StoreProtoResponse =
+        try {
+            store(args)
+        } finally {
+            /*
+             * The reactive multipart reader spills any part larger than its in-memory threshold to a
+             * temporary file on disk. Those temp files are only removed when FilePart.delete() is
+             * called, so we must delete every FilePart in the request on every exit path -- success,
+             * business error, or a validation/cast failure raised from getParams(). Skipping this is
+             * what caused /tmp/spring-multipart-* to grow unbounded over the lifetime of the pod.
+             */
+            args.request.deleteAllFileParts()
+        }
+
+    private suspend fun store(args: StoreFileRequestWrapper): StoreProtoResponse {
         val (account, permissions, objectStoreAddress, storeRawBytes, id, file, type) = getParams(args.request)
         val entity = entityManager.getEntity(KeyManagementConfigWrapper(args.entity.id, account?.keyManagementConfig))
 
@@ -56,6 +73,28 @@ class StoreFile(
         }.awaitSingle()
     }
 
+    /**
+     * Deletes the temporary file backing every [FilePart] in the request, ignoring parts that were
+     * kept in memory or already removed. Errors are swallowed so cleanup never masks the original
+     * outcome of the request.
+     */
+    private suspend fun Map<String, Part>.deleteAllFileParts() {
+        val fileParts = values.filterIsInstance<FilePart>()
+        if (fileParts.isEmpty()) {
+            return
+        }
+        /*
+         * Run inside NonCancellable so the temp files are still deleted when the request coroutine
+         * was cancelled (e.g. the client aborted the upload mid-stream) -- otherwise the suspending
+         * delete() call would immediately throw CancellationException and leak the file on disk.
+         */
+        withContext(NonCancellable) {
+            fileParts.forEach { filePart ->
+                runCatching { filePart.delete().awaitFirstOrNull() }
+            }
+        }
+    }
+
     private fun getParams(request: Map<String, Part>): Args {
         var permissions: PermissionInfo? = null
         var account: AccountInfo? = null

service/src/main/kotlin/io/provenance/api/frameworks/web/config/WebConfig.kt

@@ -1,6 +1,7 @@
 package io.provenance.api.frameworks.web.config
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.annotation.Configuration
 import org.springframework.http.codec.ServerCodecConfigurer
 import org.springframework.http.codec.json.Jackson2JsonEncoder
@@ -13,6 +14,15 @@ import org.springframework.web.reactive.config.WebFluxConfigurer
 @EnableWebFlux
 class WebConfig(
     private val objectMapper: ObjectMapper,
+    /*
+     * The maximum number of bytes of a multipart part that will be buffered in memory before the
+     * reader spills the remainder of that part to a temporary file on disk. Configurable via the
+     * MULTIPART_MAX_IN_MEMORY_SIZE environment variable (Spring relaxed binding of the
+     * multipart.max-in-memory-size property); when unset it falls back to Spring's default of
+     * 262144 bytes (256 KiB).
+     */
+    @Value("\${multipart.max-in-memory-size:262144}")
+    private val multipartMaxInMemorySize: Int,
 ) : WebFluxConfigurer {
 
     override fun configureHttpMessageCodecs(configurer: ServerCodecConfigurer) {
@@ -22,6 +32,7 @@ class WebConfig(
 
         val partReader = DefaultPartHttpMessageReader()
         partReader.setMaxHeadersSize(16 * 1024 * 1024)
+        partReader.setMaxInMemorySize(multipartMaxInMemorySize)
         val multipartReader = MultipartHttpMessageReader(partReader)
         configurer.defaultCodecs().multipartReader(multipartReader)
     }

service/src/main/kotlin/io/provenance/api/util/MultiPartExtensions.kt

@@ -1,13 +1,27 @@
 package io.provenance.api.util
 
+import org.springframework.core.io.buffer.DataBuffer
 import org.springframework.core.io.buffer.DataBufferUtils
 import org.springframework.http.codec.multipart.FilePart
 import reactor.core.publisher.Mono
-import java.nio.ByteBuffer
 
+/**
+ * Reads the full contents of this [FilePart] into a [ByteArray].
+ *
+ * The intermediate [DataBuffer] produced by [DataBufferUtils.join] is always released once its
+ * readable bytes have been copied out, including on error or cancellation. Failing to release it
+ * leaks pooled/direct buffers and increases heap and native-memory pressure under load.
+ *
+ * NOTE: callers remain responsible for invoking [FilePart.delete] to remove any temporary file that
+ * the reactive multipart reader may have spilled to disk for this part.
+ */
 fun FilePart.awaitAllBytes(): Mono<ByteArray> =
     DataBufferUtils.join(this.content()).map { dataBuffer ->
-        ByteBuffer.allocate(dataBuffer.capacity()).also { byteBuffer ->
-            dataBuffer.toByteBuffer(byteBuffer)
-        }.array()
+        try {
+            ByteArray(dataBuffer.readableByteCount()).also { bytes ->
+                dataBuffer.read(bytes)
+            }
+        } finally {
+            DataBufferUtils.release(dataBuffer)
+        }
     }

service/src/main/resources/application.properties

@@ -27,6 +27,10 @@ spring.servlet.multipart.max-file-size=10MB
 spring.servlet.multipart.max-request-size=10MB
 spring.devtools.livereload.enabled=false
 
+# Max bytes of a multipart part buffered in memory before spilling to a temp file on disk.
+# Override via the MULTIPART_MAX_IN_MEMORY_SIZE environment variable; defaults to 262144 (256 KiB).
+# multipart.max-in-memory-size=262144
+
 # Swagger settings
 springdoc.swagger-ui.enabled=true
 springdoc.swagger-ui.path=/p8e-cee-api/secure/docs/api.html