diff --git a/charts/dataplane/values.azure.yaml b/charts/dataplane/values.azure.yaml index aa9a0f5b..ebac515c 100644 --- a/charts/dataplane/values.azure.yaml +++ b/charts/dataplane/values.azure.yaml @@ -91,6 +91,13 @@ global: # Note: Key Vault must exist with appropriate access policies AZURE_KEY_VAULT_URI: "" + # 13. AZURE_STORAGE_DNS_SUFFIX - Azure Storage DNS suffix + # Default: "dfs.core.windows.net" (Azure Public Cloud, Data Lake Storage Gen2) + # Override for sovereign clouds: + # Azure China: "dfs.core.chinacloudapi.cn" + # Azure Government: "dfs.core.usgovcloudapi.net" + AZURE_STORAGE_DNS_SUFFIX: "dfs.core.windows.net" + # ---------------------------------------------------------------------------- # SECTION 2: Core Identity Configuration (REQUIRED) # ---------------------------------------------------------------------------- @@ -110,6 +117,7 @@ storage: provider: custom bucketName: '{{ .Values.global.METADATA_CONTAINER }}' enableMultiContainer: true + metadataPrefix: "abfs://{{ .Values.global.METADATA_CONTAINER }}@{{ .Values.global.AZURE_STORAGE_ACCOUNT }}.{{ .Values.global.AZURE_STORAGE_DNS_SUFFIX }}" # Custom storage configuration using stow with Azure backend custom: @@ -123,6 +131,7 @@ storage: # Leave key empty to use Workload Identity / Managed Identity authentication # For key-based auth, provide the storage account access key # key: "" + configDomainSuffix: '{{ .Values.global.AZURE_STORAGE_DNS_SUFFIX }}' # ---------------------------------------------------------------------------- # SECTION 4: Workload Identity (REQUIRED for Azure) @@ -207,7 +216,7 @@ config: operator: clusterData: # Azure Blob Storage path format (ABFS protocol for Data Lake Storage Gen2) - metadataBucketPrefix: "abfs://{{.Values.global.METADATA_CONTAINER}}@{{.Values.global.AZURE_STORAGE_ACCOUNT}}.dfs.core.windows.net" + metadataBucketPrefix: "abfs://{{.Values.global.METADATA_CONTAINER}}@{{.Values.global.AZURE_STORAGE_ACCOUNT}}.{{.Values.global.AZURE_STORAGE_DNS_SUFFIX}}" org: namespaceTemplate: '{{`{{ domain }}`}}' diff --git a/charts/dataplane/values.yaml b/charts/dataplane/values.yaml index bf41f021..d86d70e7 100644 --- a/charts/dataplane/values.yaml +++ b/charts/dataplane/values.yaml @@ -1670,7 +1670,9 @@ storage: # -- Override the metadata prefix URL used for constructing object storage paths (e.g. rawoutput-prefix, # -- metadataBucketPrefix). When set, this takes precedence over the auto-generated prefix based on the # -- storage provider. Useful for custom providers where the default s3:// scheme is incorrect. - # -- Example for Azure: "abfs://my-container@mystorageaccount.dfs.core.windows.net" + # -- Example for Azure Public: "abfs://my-container@mystorageaccount.dfs.core.windows.net" + # -- Example for Azure China: "abfs://my-container@mystorageaccount.dfs.core.chinacloudapi.cn" + # -- Example for Azure Government: "abfs://my-container@mystorageaccount.dfs.core.usgovcloudapi.net" metadataPrefix: "" # -- Define custom configurations for the object storage. Only used if the provider is set to "custom". custom: { } diff --git a/tests/generated/dataplane.azure-custom-storage-prefix.yaml b/tests/generated/dataplane.azure-custom-dns-suffix.yaml similarity index 99% rename from tests/generated/dataplane.azure-custom-storage-prefix.yaml rename to tests/generated/dataplane.azure-custom-dns-suffix.yaml index 7e646b18..a0bc6c99 100644 --- a/tests/generated/dataplane.azure-custom-storage-prefix.yaml +++ b/tests/generated/dataplane.azure-custom-dns-suffix.yaml @@ -557,6 +557,7 @@ data: stow: config: account: 'teststorageaccount' + configDomainSuffix: 'dfs.core.chinacloudapi.cn' kind: azure type: stow enable-multicontainer: true @@ -633,7 +634,7 @@ data: bucketRegion: 'us-east-1' cloudHostName: 'test.dataplane.union.ai' gcpProjectId: '' - metadataBucketPrefix: abfs://test-metadata-container@teststorageaccount.dfs.core.windows.net + metadataBucketPrefix: abfs://test-metadata-container@teststorageaccount.dfs.core.chinacloudapi.cn userRole: 'test-worker-client-id' userRoleKey: 'azure.workload.identity/client-id' # -- storageType is only used when syncClusterConfig is enabled. It is intentionally disabled and it should not be used. @@ -643,6 +644,7 @@ data: stow: config: account: 'teststorageaccount' + configDomainSuffix: 'dfs.core.chinacloudapi.cn' kind: azure type: stow collectUsages: @@ -697,6 +699,7 @@ data: stow: config: account: 'teststorageaccount' + configDomainSuffix: 'dfs.core.chinacloudapi.cn' kind: azure type: stow enable-multicontainer: true @@ -712,6 +715,7 @@ data: stow: config: account: 'teststorageaccount' + configDomainSuffix: 'dfs.core.chinacloudapi.cn' kind: azure type: stow image-builder.buildkit-uri: "tcp://union-operator-buildkit.union.svc.cluster.local:1234" @@ -782,7 +786,7 @@ data: rate: 10 type: bucket type: batch - rawoutput-prefix: 'abfs://test-metadata-container@teststorageaccount.dfs.core.windows.net' + rawoutput-prefix: 'abfs://test-metadata-container@teststorageaccount.dfs.core.chinacloudapi.cn' workers: 4 workflow-reeval-duration: 30s webhook: @@ -901,6 +905,7 @@ data: stow: config: account: 'teststorageaccount' + configDomainSuffix: 'dfs.core.chinacloudapi.cn' kind: azure type: stow enable-multicontainer: true @@ -2890,7 +2895,7 @@ spec: template: metadata: annotations: - configChecksum: "35386edbc0829a2990c4f836f17b26614cbcce81c3839deb56fa24190fa5d8b" + configChecksum: "687582e700be57db5ab5f1d6841f53794c9d80f2a36abf7893d3e7ae0948d39" labels: azure.workload.identity/use: "true" @@ -2992,7 +2997,7 @@ spec: template: metadata: annotations: - configChecksum: "90387a33bf24bf2fd8880aa120dcf940fc6db9dffd35027452107c23e4e7dbb" + configChecksum: "4bf81f7d17029d463673335da2dcd294aa1ff2825715d129846024cfd48b86c" labels: @@ -3130,7 +3135,7 @@ spec: template: metadata: annotations: - configChecksum: "90387a33bf24bf2fd8880aa120dcf940fc6db9dffd35027452107c23e4e7dbb" + configChecksum: "4bf81f7d17029d463673335da2dcd294aa1ff2825715d129846024cfd48b86c" labels: @@ -3249,7 +3254,7 @@ spec: platform.union.ai/service-group: release-name app.kubernetes.io/managed-by: Helm annotations: - configChecksum: "56e2f2952ca048e0fb346ff74e7f709def08ef79fca833e65a8564b7160c1de" + configChecksum: "21f5542313bf64f062d18998288b41f518779b023227d201d5bbb84d3c1874d" spec: securityContext: @@ -3404,7 +3409,7 @@ spec: template: metadata: annotations: - configChecksum: "56e2f2952ca048e0fb346ff74e7f709def08ef79fca833e65a8564b7160c1de" + configChecksum: "21f5542313bf64f062d18998288b41f518779b023227d201d5bbb84d3c1874d" labels: diff --git a/tests/generated/dataplane.azure.yaml b/tests/generated/dataplane.azure.yaml index be53bbde..97b57188 100644 --- a/tests/generated/dataplane.azure.yaml +++ b/tests/generated/dataplane.azure.yaml @@ -1400,6 +1400,7 @@ data: stow: config: account: 'test-storage-account' + configDomainSuffix: 'dfs.core.windows.net' kind: azure type: stow enable-multicontainer: true @@ -1486,6 +1487,7 @@ data: stow: config: account: 'test-storage-account' + configDomainSuffix: 'dfs.core.windows.net' kind: azure type: stow collectUsages: @@ -1540,6 +1542,7 @@ data: stow: config: account: 'test-storage-account' + configDomainSuffix: 'dfs.core.windows.net' kind: azure type: stow enable-multicontainer: true @@ -1555,6 +1558,7 @@ data: stow: config: account: 'test-storage-account' + configDomainSuffix: 'dfs.core.windows.net' kind: azure type: stow image-builder.buildkit-uri: "tcp://union-operator-buildkit.union.svc.cluster.local:1234" @@ -1625,7 +1629,7 @@ data: rate: 10 type: bucket type: batch - rawoutput-prefix: 's3://' + rawoutput-prefix: 'abfs://test-metadata-container@test-storage-account.dfs.core.windows.net' workers: 4 workflow-reeval-duration: 30s webhook: @@ -1744,6 +1748,7 @@ data: stow: config: account: 'test-storage-account' + configDomainSuffix: 'dfs.core.windows.net' kind: azure type: stow enable-multicontainer: true @@ -4219,7 +4224,7 @@ spec: template: metadata: annotations: - configChecksum: "59f05c12acb770d6e9a63b282585a51c913ce57d760c0671e58abea7bdbd3e8" + configChecksum: "363f69eddeba91a77891cd184fdffc9728687eb85e38457055f6efe3f2ab1b8" labels: azure.workload.identity/use: "true" @@ -4321,7 +4326,7 @@ spec: template: metadata: annotations: - configChecksum: "561a9278d8384858f2360066d5d1a83ae5777730ed93574ee6d1a56f900b83a" + configChecksum: "19fd3357f89a86221a90957ebf49f4650bd7bc4445b22561c1a869a1ff45d82" labels: @@ -4459,7 +4464,7 @@ spec: template: metadata: annotations: - configChecksum: "561a9278d8384858f2360066d5d1a83ae5777730ed93574ee6d1a56f900b83a" + configChecksum: "19fd3357f89a86221a90957ebf49f4650bd7bc4445b22561c1a869a1ff45d82" labels: @@ -4578,7 +4583,7 @@ spec: platform.union.ai/service-group: release-name app.kubernetes.io/managed-by: Helm annotations: - configChecksum: "d4bcd8601d583eb3f54b7012d472f044f8029e3e70d2eaeeab0b6d57c299f23" + configChecksum: "79826ed1dcbf442db6e5555095e4539db8e6976a71bd71c1fd9d2a69ecb3753" spec: securityContext: @@ -4733,7 +4738,7 @@ spec: template: metadata: annotations: - configChecksum: "d4bcd8601d583eb3f54b7012d472f044f8029e3e70d2eaeeab0b6d57c299f23" + configChecksum: "79826ed1dcbf442db6e5555095e4539db8e6976a71bd71c1fd9d2a69ecb3753" labels: diff --git a/tests/values/dataplane.azure-custom-storage-prefix.yaml b/tests/values/dataplane.azure-custom-dns-suffix.yaml similarity index 85% rename from tests/values/dataplane.azure-custom-storage-prefix.yaml rename to tests/values/dataplane.azure-custom-dns-suffix.yaml index ced450cd..64b5f5a1 100644 --- a/tests/values/dataplane.azure-custom-storage-prefix.yaml +++ b/tests/values/dataplane.azure-custom-dns-suffix.yaml @@ -1,5 +1,7 @@ -# Test: custom storage.metadataPrefix overrides the auto-generated s3:// prefix -# for Azure custom storage providers using ABFS protocol. +# Test: Azure sovereign cloud deployment. +# Only AZURE_STORAGE_DNS_SUFFIX differs from the standard Azure test. +# Validates that the DNS suffix flows through to metadataPrefix, +# configDomainSuffix, and metadataBucketPrefix. global: UNION_CONTROL_PLANE_HOST: "test.dataplane.union.ai" @@ -14,6 +16,8 @@ global: AZURE_BACKEND_CLIENT_ID: "test-backend-client-id" AZURE_WORKER_CLIENT_ID: "test-worker-client-id" AZURE_KEY_VAULT_URI: "test-azure-key-vault-uri" + # Custom DNS suffix for sovereign cloud (e.g., Azure Government, Azure China) + AZURE_STORAGE_DNS_SUFFIX: "dfs.core.chinacloudapi.cn" provider: azure @@ -23,11 +27,11 @@ secrets: clientSecret: "test-client-secret-value" create: true -# Custom Azure storage with explicit metadataPrefix +# Azure storage with custom DNS suffix - only AZURE_STORAGE_DNS_SUFFIX changes storage: provider: custom bucketName: test-metadata-container - metadataPrefix: "abfs://test-metadata-container@teststorageaccount.dfs.core.windows.net" + metadataPrefix: "abfs://{{ .Values.global.METADATA_CONTAINER }}@{{ .Values.global.AZURE_STORAGE_ACCOUNT }}.{{ .Values.global.AZURE_STORAGE_DNS_SUFFIX }}" enableMultiContainer: true custom: container: '{{ .Values.global.METADATA_CONTAINER }}' @@ -36,6 +40,7 @@ storage: kind: azure config: account: '{{ .Values.global.AZURE_STORAGE_ACCOUNT }}' + configDomainSuffix: '{{ .Values.global.AZURE_STORAGE_DNS_SUFFIX }}' additionalServiceAccountAnnotations: azure.workload.identity/client-id: "{{ .Values.global.AZURE_BACKEND_CLIENT_ID }}" @@ -64,7 +69,7 @@ config: operator: clusterData: - metadataBucketPrefix: "abfs://{{.Values.global.METADATA_CONTAINER}}@{{.Values.global.AZURE_STORAGE_ACCOUNT}}.dfs.core.windows.net" + metadataBucketPrefix: "abfs://{{.Values.global.METADATA_CONTAINER}}@{{.Values.global.AZURE_STORAGE_ACCOUNT}}.{{.Values.global.AZURE_STORAGE_DNS_SUFFIX}}" org: namespaceTemplate: '{{`{{ domain }}`}}' diff --git a/tests/values/dataplane.azure.yaml b/tests/values/dataplane.azure.yaml index 495564e3..f7ac742e 100644 --- a/tests/values/dataplane.azure.yaml +++ b/tests/values/dataplane.azure.yaml @@ -14,6 +14,7 @@ global: AZURE_BACKEND_CLIENT_ID: "test-backend-client-id" AZURE_WORKER_CLIENT_ID: "test-worker-client-id" AZURE_KEY_VAULT_URI: "test-azure-key-vault-uri" + AZURE_STORAGE_DNS_SUFFIX: "dfs.core.windows.net" # ---------------------------------------------------------------------------- # SECTION 2: Core Identity Configuration (REQUIRED) @@ -33,6 +34,7 @@ provider: azure storage: provider: custom enableMultiContainer: true + metadataPrefix: "abfs://{{ .Values.global.METADATA_CONTAINER }}@{{ .Values.global.AZURE_STORAGE_ACCOUNT }}.{{ .Values.global.AZURE_STORAGE_DNS_SUFFIX }}" # Custom storage configuration using stow with Azure backend custom: @@ -41,11 +43,8 @@ storage: stow: kind: azure config: - # Storage account name account: '{{ .Values.global.AZURE_STORAGE_ACCOUNT }}' - # Leave key empty to use Workload Identity / Managed Identity authentication - # For key-based auth, provide the storage account access key - # key: "" + configDomainSuffix: '{{ .Values.global.AZURE_STORAGE_DNS_SUFFIX }}' # ---------------------------------------------------------------------------- # SECTION 4: Workload Identity (REQUIRED for Azure) @@ -127,7 +126,7 @@ config: operator: clusterData: # Azure Blob Storage path format (ABFS protocol for Data Lake Storage Gen2) - metadataBucketPrefix: "abfs://{{.Values.global.METADATA_CONTAINER}}@{{.Values.global.AZURE_STORAGE_ACCOUNT}}.dfs.core.windows.net" + metadataBucketPrefix: "abfs://{{.Values.global.METADATA_CONTAINER}}@{{.Values.global.AZURE_STORAGE_ACCOUNT}}.{{.Values.global.AZURE_STORAGE_DNS_SUFFIX}}" org: namespaceTemplate: '{{`{{ domain }}`}}'