From 28cdb1ec430789f3703e067106eba9af4406f20b Mon Sep 17 00:00:00 2001 From: Varshi Bachu Date: Wed, 16 Sep 2026 00:01:39 -0700 Subject: [PATCH 1/3] initial commit --- .github/workflows/build-validation.yml | 17 ++++- CHANGELOG.md | 9 ++- README.md | 11 +++- azdevops-pipeline/build-release-artifacts.yml | 4 +- azure-blob-payloads/README.md | 63 ++++++++++++++++++ azurefunctions/build.gradle | 2 +- azuremanaged/build.gradle | 2 +- client/build.gradle | 2 +- eng/ci/release.yml | 66 +++++++++++++++++-- eng/templates/build.yml | 4 +- 10 files changed, 161 insertions(+), 19 deletions(-) create mode 100644 azure-blob-payloads/README.md diff --git a/.github/workflows/build-validation.yml b/.github/workflows/build-validation.yml index 4ca98e6d..e1b66fa9 100644 --- a/.github/workflows/build-validation.yml +++ b/.github/workflows/build-validation.yml @@ -142,6 +142,14 @@ jobs: name: Integration test report path: client/build/reports/tests/integrationTest + - name: Archive large payload integration test report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: Large payload integration test report + path: azure-blob-payloads/build/reports/tests/integrationTest + if-no-files-found: ignore + - name: Archive export history integration test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -153,8 +161,13 @@ jobs: - name: Upload JAR output uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: Package - path: client/build/libs + name: Packages + path: | + client/build/libs + azurefunctions/build/libs + azuremanaged/build/libs + azure-blob-payloads/build/libs + exporthistory/build/libs - name: Fail the job if tests failed if: env.TEST_FAILED == 'true' diff --git a/CHANGELOG.md b/CHANGELOG.md index f1deaf3b..4c6d8754 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ ## Unreleased + +## v1.10.0 +* Enable custom exception providers for Durable Functions Java activities ([#294](https://github.com/microsoft/durabletask-java/pull/294)) * Add the `exporthistory` module for durable, checkpointed export of terminal orchestration history to Azure Blob Storage ([#293](https://github.com/microsoft/durabletask-java/pull/293)) * Add client APIs to list terminal instance IDs by completion time (`listInstanceIds`) and read orchestration history (`getOrchestrationHistory`) ([#292](https://github.com/microsoft/durabletask-java/pull/292)) -* Add `createReplaySafeLogger` to suppress orchestration log output during replay ([#295](https://github.com/microsoft/durabletask-java/pull/295)). -* Add `getParentInstance()` API to `TaskOrchestrationContext` for discovering parent orchestration info ([#284](https://github.com/microsoft/durabletask-java/pull/284)) +* Add `createReplaySafeLogger` to suppress orchestration log output during replay ([#295](https://github.com/microsoft/durabletask-java/pull/295)) +* Add `getParent()` API to `TaskOrchestrationContext` for discovering parent orchestration info ([#284](https://github.com/microsoft/durabletask-java/pull/284)) +* Add the `azure-blob-payloads` module for externalizing large orchestration payloads to Azure Blob Storage ([#280](https://github.com/microsoft/durabletask-java/pull/280)) +* Add inner failure details and custom exception properties to `FailureDetails` ([#263](https://github.com/microsoft/durabletask-java/pull/263)) ## v1.9.0 * Fix entity locking deserialization and add Jackson support for EntityInstanceId/EntityMetadata ([#281](https://github.com/microsoft/durabletask-java/pull/281)) diff --git a/README.md b/README.md index 7f5b8b53..97fd19b8 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,10 @@ Logger logger = ctx.createReplaySafeLogger(executionContext.getLogger()); Replay-safe logging suppresses calls made while replaying; it does not guarantee exactly-once log delivery across failed or retried live orchestration turns. +### Externalized large payloads + +The [`durabletask-azure-blob-payloads`](azure-blob-payloads/README.md) module transparently stores orchestration payloads that exceed a configurable threshold in Azure Blob Storage. Configure the same storage container on the client and worker to exchange large inputs, outputs, activity payloads, and orchestration actions without exceeding gRPC message limits. + ### Reliable fan-out / fan-in orchestration pattern ```java @@ -102,8 +106,11 @@ The following packages are produced from this repo. | Package | Latest version | | - | - | -| Durable Task - Client | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-client?label=durabletask-client)](https://mvnrepository.com/artifact/com.microsoft/durabletask-client/1.0.0) | -| Durable Task - Azure Functions | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-azure-functions?label=durabletask-azure-functions)](https://mvnrepository.com/artifact/com.microsoft/durabletask-azure-functions/1.0.1) | +| Durable Task - Client | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-client?label=durabletask-client)](https://mvnrepository.com/artifact/com.microsoft/durabletask-client) | +| Durable Task - Azure Functions | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-azure-functions?label=durabletask-azure-functions)](https://mvnrepository.com/artifact/com.microsoft/durabletask-azure-functions) | +| Durable Task - Azure Managed | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-azuremanaged?label=durabletask-azuremanaged)](https://mvnrepository.com/artifact/com.microsoft/durabletask-azuremanaged) | +| [Durable Task - Azure Blob Payloads](azure-blob-payloads/README.md) | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-azure-blob-payloads?label=durabletask-azure-blob-payloads)](https://mvnrepository.com/artifact/com.microsoft/durabletask-azure-blob-payloads) | +| [Durable Task - Export History](exporthistory/README.md) | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-exporthistory?label=durabletask-exporthistory)](https://mvnrepository.com/artifact/com.microsoft/durabletask-exporthistory) | ## Getting started with Azure Functions diff --git a/azdevops-pipeline/build-release-artifacts.yml b/azdevops-pipeline/build-release-artifacts.yml index 71c50a30..ab3c4ed3 100644 --- a/azdevops-pipeline/build-release-artifacts.yml +++ b/azdevops-pipeline/build-release-artifacts.yml @@ -25,7 +25,7 @@ steps: publishJUnitResults: false tasks: clean assemble options: '--init-script init.gradle' - displayName: Assemble durabletask-client and durabletask-azure-functions + displayName: Assemble all publishable Durable Task Java packages # the secring.gpg file is required to sign the artifacts, it's generated from GnuPG, and it's stored in the library of the durabletaskframework ADO - task: DownloadSecureFile@1 @@ -44,7 +44,7 @@ steps: jdkArchitectureOption: 'x64' tasks: publish options: '--init-script init.gradle -Psigning.keyId=$(gpgSignKey) -Psigning.password=$(gpgSignPassword) -Psigning.secretKeyRingFile=$(gpgSecretFile.secureFilePath)' - displayName: Publish durabletask-client and durabletask-azure-functions + displayName: Stage all publishable Durable Task Java packages - task: CopyFiles@2 displayName: 'Copy publish file to Artifact Staging Directory' diff --git a/azure-blob-payloads/README.md b/azure-blob-payloads/README.md new file mode 100644 index 00000000..e1977938 --- /dev/null +++ b/azure-blob-payloads/README.md @@ -0,0 +1,63 @@ +# Durable Task Azure Blob Payloads for Java + +The `durabletask-azure-blob-payloads` module transparently externalizes large Durable Task payloads to Azure Blob Storage. Payloads below the configured threshold continue to travel inline; larger payloads are replaced with opaque blob references and restored before user code receives them. + +## Install + +Add the module alongside the Durable Task client and your backend-specific extension: + +```groovy +implementation 'com.microsoft:durabletask-azure-blob-payloads:1.0.0' +``` + +The module includes Azure Blob Storage support. Applications that use managed identity should also add `com.azure:azure-identity`. + +## Usage + +Configure the same storage location for every client and worker that exchanges externalized payloads: + +```java +LargePayloadStorageOptions payloadOptions = new LargePayloadStorageOptions() + .setConnectionString(System.getenv("PAYLOAD_STORAGE_CONNECTION_STRING")) + .setContainerName("durabletask-payloads") + .setThresholdBytes(900_000); + +PayloadStore payloadStore = new BlobPayloadStore(payloadOptions); + +DurableTaskGrpcClientBuilder clientBuilder = new DurableTaskGrpcClientBuilder(); +DurableTaskSchedulerClientExtensions.useDurableTaskScheduler(clientBuilder, schedulerConnectionString); +LargePayloadClientExtensions.useExternalizedPayloads(clientBuilder, payloadStore, payloadOptions); + +DurableTaskGrpcWorkerBuilder workerBuilder = new DurableTaskGrpcWorkerBuilder(); +DurableTaskSchedulerWorkerExtensions.useDurableTaskScheduler(workerBuilder, schedulerConnectionString); +LargePayloadWorkerExtensions.useExternalizedPayloads(workerBuilder, payloadStore, payloadOptions); +``` + +For identity-based authentication, configure an account URI and `TokenCredential` instead of a connection string: + +```java +LargePayloadStorageOptions payloadOptions = new LargePayloadStorageOptions() + .setAccountUri(URI.create("https://.blob.core.windows.net")) + .setCredential(new DefaultAzureCredentialBuilder().build()); +``` + +## Defaults and limits + +- Payloads of at least 900,000 bytes are externalized by default. +- The threshold can be configured up to 1 MiB. +- The default maximum externalized payload size is 10 MiB. +- Payloads are gzip-compressed by default. +- The default container name is `durabletask-payloads`. +- The container is created automatically on the first upload. + +The storage identity needs permission to create the container and read and write blobs. Payload blobs are retained after orchestration processing, so configure an Azure Storage lifecycle policy appropriate for the application's retention requirements. + +## Sample + +See [`LargePayloadSample`](../samples/src/main/java/io/durabletask/samples/LargePayloadSample.java): + +```text +./gradlew :samples:runLargePayloadSample +``` + +The sample uses the DTS emulator and Azurite and verifies a payload larger than 1 MiB through a client, orchestration, and activity round trip. diff --git a/azurefunctions/build.gradle b/azurefunctions/build.gradle index ac751281..d8cef822 100644 --- a/azurefunctions/build.gradle +++ b/azurefunctions/build.gradle @@ -6,7 +6,7 @@ plugins { } group 'com.microsoft' -version = '1.9.0' +version = '1.10.0' archivesBaseName = 'durabletask-azure-functions' def protocVersion = '3.25.8' diff --git a/azuremanaged/build.gradle b/azuremanaged/build.gradle index 25842223..de0c798b 100644 --- a/azuremanaged/build.gradle +++ b/azuremanaged/build.gradle @@ -17,7 +17,7 @@ plugins { archivesBaseName = 'durabletask-azuremanaged' group 'com.microsoft' -version = '1.9.0' +version = '1.10.0' def grpcVersion = '1.78.0' def azureCoreVersion = '1.57.1' diff --git a/client/build.gradle b/client/build.gradle index d68b57de..90d3a2e8 100644 --- a/client/build.gradle +++ b/client/build.gradle @@ -10,7 +10,7 @@ plugins { } group 'com.microsoft' -version = '1.9.0' +version = '1.10.0' archivesBaseName = 'durabletask-client' def grpcVersion = '1.78.0' diff --git a/eng/ci/release.yml b/eng/ci/release.yml index 408a7404..d345f2fa 100644 --- a/eng/ci/release.yml +++ b/eng/ci/release.yml @@ -23,7 +23,7 @@ extends: stages: - stage: release jobs: - - job: durabletask_azuremanaged + - job: durabletask_client templateContext: type: releaseJob isProduction: true @@ -36,19 +36,20 @@ extends: steps: - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 - displayName: 'Release durabletask-azuremanaged' + displayName: 'Release durabletask-client' inputs: connectedservicename: 'dtfx-internal-esrp-prod' usemanagedidentity: true keyvaultname: 'durable-esrp-akv' signcertname: 'dts-esrp-cert' clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' - folderlocation: '$(System.DefaultWorkingDirectory)/drop/durabletask-azuremanaged' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/durabletask-client' owners: 'wangbill@microsoft.com' approvers: 'kaibocai@microsoft.com' mainpublisher: 'durabletask-java' - - job: durabletask_client + - job: durabletask_azuremanaged + dependsOn: durabletask_client templateContext: type: releaseJob isProduction: true @@ -61,19 +62,20 @@ extends: steps: - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 - displayName: 'Release durabletask-client' + displayName: 'Release durabletask-azuremanaged' inputs: connectedservicename: 'dtfx-internal-esrp-prod' usemanagedidentity: true keyvaultname: 'durable-esrp-akv' signcertname: 'dts-esrp-cert' clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' - folderlocation: '$(System.DefaultWorkingDirectory)/drop/durabletask-client' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/durabletask-azuremanaged' owners: 'wangbill@microsoft.com' approvers: 'kaibocai@microsoft.com' mainpublisher: 'durabletask-java' - job: durabletask_azure_functions + dependsOn: durabletask_client templateContext: type: releaseJob isProduction: true @@ -97,3 +99,55 @@ extends: owners: 'wangbill@microsoft.com' approvers: 'kaibocai@microsoft.com' mainpublisher: 'durabletask-java' + + - job: durabletask_azure_blob_payloads + dependsOn: durabletask_client + templateContext: + type: releaseJob + isProduction: true + inputs: + # Declare inputs to be released here to ensure they receive relevant checks + - input: pipelineArtifact + pipeline: DurableTaskJavaBuildPipeline + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + steps: + - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 + displayName: 'Release durabletask-azure-blob-payloads' + inputs: + connectedservicename: 'dtfx-internal-esrp-prod' + usemanagedidentity: true + keyvaultname: 'durable-esrp-akv' + signcertname: 'dts-esrp-cert' + clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/durabletask-azure-blob-payloads' + owners: 'wangbill@microsoft.com' + approvers: 'kaibocai@microsoft.com' + mainpublisher: 'durabletask-java' + + - job: durabletask_exporthistory + dependsOn: durabletask_client + templateContext: + type: releaseJob + isProduction: true + inputs: + # Declare inputs to be released here to ensure they receive relevant checks + - input: pipelineArtifact + pipeline: DurableTaskJavaBuildPipeline + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + steps: + - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 + displayName: 'Release durabletask-exporthistory' + inputs: + connectedservicename: 'dtfx-internal-esrp-prod' + usemanagedidentity: true + keyvaultname: 'durable-esrp-akv' + signcertname: 'dts-esrp-cert' + clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/durabletask-exporthistory' + owners: 'wangbill@microsoft.com' + approvers: 'kaibocai@microsoft.com' + mainpublisher: 'durabletask-java' diff --git a/eng/templates/build.yml b/eng/templates/build.yml index 33aa780b..3e381a16 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -25,7 +25,7 @@ jobs: publishJUnitResults: false tasks: clean assemble options: '--init-script init.gradle' - displayName: Assemble durabletask-client and durabletask-azure-functions and durabletask-azuremanaged + displayName: Assemble all publishable Durable Task Java packages # the secring.gpg file is required to sign the artifacts, it's generated from GnuPG, and it's stored in the library of the durabletaskframework ADO - task: DownloadSecureFile@1 @@ -44,7 +44,7 @@ jobs: jdkArchitectureOption: 'x64' tasks: publish options: '--init-script init.gradle -Psigning.keyId=$(gpgSignKey) -Psigning.password=$(gpgSignPassword) -Psigning.secretKeyRingFile=$(gpgSecretFile.secureFilePath)' - displayName: Publish durabletask-client and durabletask-azure-functions and durabletask-azuremanaged + displayName: Stage all publishable Durable Task Java packages - task: CopyFiles@2 displayName: 'Copy publish file to Artifact Staging Directory' From e4253f15025d0c18ff9e7038547d63902f498713 Mon Sep 17 00:00:00 2001 From: Varshi Bachu Date: Wed, 16 Sep 2026 00:33:54 -0700 Subject: [PATCH 2/3] updated versions --- azure-blob-payloads/README.md | 2 +- azure-blob-payloads/build.gradle | 2 +- exporthistory/README.md | 4 ++-- exporthistory/build.gradle | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-blob-payloads/README.md b/azure-blob-payloads/README.md index e1977938..aeb84728 100644 --- a/azure-blob-payloads/README.md +++ b/azure-blob-payloads/README.md @@ -7,7 +7,7 @@ The `durabletask-azure-blob-payloads` module transparently externalizes large Du Add the module alongside the Durable Task client and your backend-specific extension: ```groovy -implementation 'com.microsoft:durabletask-azure-blob-payloads:1.0.0' +implementation 'com.microsoft:durabletask-azure-blob-payloads:1.10.0' ``` The module includes Azure Blob Storage support. Applications that use managed identity should also add `com.azure:azure-identity`. diff --git a/azure-blob-payloads/build.gradle b/azure-blob-payloads/build.gradle index 61d033ed..8b9e4c16 100644 --- a/azure-blob-payloads/build.gradle +++ b/azure-blob-payloads/build.gradle @@ -6,7 +6,7 @@ plugins { } group 'com.microsoft' -version = '1.0.0' +version = '1.10.0' archivesBaseName = 'durabletask-azure-blob-payloads' def grpcVersion = '1.78.0' diff --git a/exporthistory/README.md b/exporthistory/README.md index 83b044af..c4f137c3 100644 --- a/exporthistory/README.md +++ b/exporthistory/README.md @@ -7,14 +7,14 @@ This module is at parity with the .NET `Microsoft.DurableTask.ExportHistory` (pr entity + orchestrator that pages terminal instances by completion window, fans out per-instance export activities, and uploads serialized history (gzipped JSONL by default) to a customer-owned blob container. -> **Status:** preview (`0.1.0`). +> **Status:** preview (`1.10.0-preview.1`). ## Install Add the module dependency alongside the core `client` (and your Durable Task Scheduler extension): ```groovy -implementation 'com.microsoft:durabletask-exporthistory:0.1.0' +implementation 'com.microsoft:durabletask-exporthistory:1.10.0-preview.1' ``` The export activities upload to Azure Blob Storage via `azure-storage-blob`. If you authenticate with a managed diff --git a/exporthistory/build.gradle b/exporthistory/build.gradle index e24fa0c6..fd4a596a 100644 --- a/exporthistory/build.gradle +++ b/exporthistory/build.gradle @@ -6,7 +6,7 @@ plugins { } group 'com.microsoft' -version = '0.1.0' +version = '1.10.0-preview.1' archivesBaseName = 'durabletask-exporthistory' def grpcVersion = '1.78.0' From 9ab4bc8fd5f193b5a230af8105076afd4c0f788e Mon Sep 17 00:00:00 2001 From: Varshitha Bachu Date: Wed, 16 Sep 2026 10:12:57 -0700 Subject: [PATCH 3/3] Update CHANGELOG with new features and APIs Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c6d8754..f25769c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * Add the `exporthistory` module for durable, checkpointed export of terminal orchestration history to Azure Blob Storage ([#293](https://github.com/microsoft/durabletask-java/pull/293)) * Add client APIs to list terminal instance IDs by completion time (`listInstanceIds`) and read orchestration history (`getOrchestrationHistory`) ([#292](https://github.com/microsoft/durabletask-java/pull/292)) * Add `createReplaySafeLogger` to suppress orchestration log output during replay ([#295](https://github.com/microsoft/durabletask-java/pull/295)) -* Add `getParent()` API to `TaskOrchestrationContext` for discovering parent orchestration info ([#284](https://github.com/microsoft/durabletask-java/pull/284)) +* Add `getParentInstance()` API to `TaskOrchestrationContext` for discovering parent orchestration info ([#284](https://github.com/microsoft/durabletask-java/pull/284)) * Add the `azure-blob-payloads` module for externalizing large orchestration payloads to Azure Blob Storage ([#280](https://github.com/microsoft/durabletask-java/pull/280)) * Add inner failure details and custom exception properties to `FailureDetails` ([#263](https://github.com/microsoft/durabletask-java/pull/263))