From 3a2b7a79406459bd129067dbc008617e1bac0bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Mon, 15 Dec 2025 11:21:18 +0100 Subject: [PATCH 1/6] feat: storage-cli consumed by capi - templates updated to be compatible with storage-cli. - All providers supported by storage-cli added into templates - package for storage-cli created - blob storage-cli dev version added to be tested --- config/blobs.yml | 7 +- .../storage_cli_config_buildpacks.json.erb | 69 +++++++++++++++--- .../storage_cli_config_droplets.json.erb | 71 ++++++++++++++++--- .../storage_cli_config_packages.json.erb | 69 +++++++++++++++--- .../storage_cli_config_resource_pool.json.erb | 69 +++++++++++++++--- jobs/cloud_controller_clock/spec | 2 +- .../storage_cli_config_buildpacks.json.erb | 69 +++++++++++++++--- .../storage_cli_config_droplets.json.erb | 71 ++++++++++++++++--- .../storage_cli_config_packages.json.erb | 69 +++++++++++++++--- .../storage_cli_config_resource_pool.json.erb | 69 +++++++++++++++--- jobs/cloud_controller_ng/spec | 2 +- .../storage_cli_config_buildpacks.json.erb | 69 +++++++++++++++--- .../storage_cli_config_droplets.json.erb | 71 ++++++++++++++++--- .../storage_cli_config_packages.json.erb | 69 +++++++++++++++--- .../storage_cli_config_resource_pool.json.erb | 69 +++++++++++++++--- jobs/cloud_controller_worker/spec | 2 +- .../storage_cli_config_buildpacks.json.erb | 69 +++++++++++++++--- .../storage_cli_config_droplets.json.erb | 71 ++++++++++++++++--- .../storage_cli_config_packages.json.erb | 69 +++++++++++++++--- .../storage_cli_config_resource_pool.json.erb | 69 +++++++++++++++--- packages/azure-storage-cli/README.md | 9 --- packages/azure-storage-cli/packaging | 5 -- packages/azure-storage-cli/spec | 4 -- packages/storage-cli/README.md | 9 +++ packages/storage-cli/packaging | 6 ++ packages/storage-cli/spec | 4 ++ 26 files changed, 973 insertions(+), 189 deletions(-) delete mode 100644 packages/azure-storage-cli/README.md delete mode 100644 packages/azure-storage-cli/packaging delete mode 100644 packages/azure-storage-cli/spec create mode 100644 packages/storage-cli/README.md create mode 100644 packages/storage-cli/packaging create mode 100644 packages/storage-cli/spec diff --git a/config/blobs.yml b/config/blobs.yml index 84d5bb2a6e..990108d7ec 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -1,7 +1,3 @@ -azure-storage-cli/azure-storage-cli-linux-amd64: - size: 7143608 - object_id: ceb2994f-cbe8-4695-413c-33785a0b6322 - sha: sha256:246944046f7e2f919965466055e957c30aeecf13012059c624d12ab04f48f822 expat/expat-2.5.0.tar.bz2: size: 569205 object_id: 970ccd16-75ac-4c55-5280-c00c4aa8f6cc @@ -98,6 +94,9 @@ postgres/postgresql-11.22.tar.gz: size: 26826810 object_id: d1f8d34c-b438-44e7-7672-5daea8a6da66 sha: sha256:6445a4e1533c1e8bb616d4a3784bdc4c0226b541f6f0c8d996d9f27d581d49c3 +storage-cli/storage-cli-dev.1.0.0-linux-amd64: + size: 59856924 + sha: sha256:08b5e97f7b1f59254df7d41741106c920a2ce65055f6fd90ddcdcccba7f20c32 valkey/7.2.11.tar.gz: size: 3441067 object_id: ac3c2823-80c0-41e9-78e4-e42014aad04a diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb index 36ba5f755a..e5c9c9b41a 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpacks bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,19 +24,68 @@ l = link("cloud_controller_internal") scope = "cc.buildpacks.connection_config" provider = l.p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options["container_name"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]=l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = l.p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -44,7 +94,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb index 363a7011f6..edaf922129 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,28 +24,76 @@ l = link("cloud_controller_internal") scope = "cc.droplets.connection_config" provider = l.p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]=l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin - custom = l.p("cc.droplets.connection_config.custom", {}) + custom = l.p("#{scope}.custom", {}) if custom.respond_to?(:each) custom.each { |k, v| add(options, k.to_s, v) } end rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb index 4de210731c..5d210faf28 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,19 +24,68 @@ l = link("cloud_controller_internal") scope = "cc.packages.connection_config" provider = l.p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]= l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = l.p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -44,7 +94,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb index bbf336fcfb..0730a3e855 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,19 +24,68 @@ l = link("cloud_controller_internal") scope = "cc.resource_pool.connection_config" provider = l.p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]=l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = l.p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -44,7 +94,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index 5b8a7d3134..b046c7223f 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -31,7 +31,7 @@ templates: storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json packages: - - azure-storage-cli + - storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index 400ccf3d01..3872194bb2 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpack bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.buildpacks.connection_config" provider = p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index 60e46c9f30..362857ce03 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,76 @@ end scope = "cc.droplets.connection_config" provider = p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin - custom = p("cc.droplets.connection_config.custom", {}) + custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) custom.each { |k, v| add(options, k.to_s, v) } end rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index f11ec11221..a958dc06a8 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.packages.connection_config" provider = p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index 9b3f7f40d9..af231116a3 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.resource_pool.connection_config" provider = p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index e44390f239..35cdde188e 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -70,7 +70,7 @@ templates: cloud_controller_local_worker_override.yml.erb: config/cloud_controller_local_worker_override.yml packages: - - azure-storage-cli + - storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index 400ccf3d01..e0c3b11c0c 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpacs bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.buildpacks.connection_config" provider = p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index 60e46c9f30..5d07cfeda8 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,76 @@ end scope = "cc.droplets.connection_config" provider = p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin - custom = p("cc.droplets.connection_config.custom", {}) + custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) custom.each { |k, v| add(options, k.to_s, v) } end rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index f11ec11221..06396e9b8e 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.packages.connection_config" provider = p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index 9b3f7f40d9..af231116a3 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.resource_pool.connection_config" provider = p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 21f5c555f0..c99806be0c 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -41,7 +41,7 @@ templates: storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json packages: - - azure-storage-cli + - storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 400ccf3d01..1d05a9f96b 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpacks bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.buildpacks.connection_config" provider = p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index 60e46c9f30..362857ce03 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,76 @@ end scope = "cc.droplets.connection_config" provider = p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin - custom = p("cc.droplets.connection_config.custom", {}) + custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) custom.each { |k, v| add(options, k.to_s, v) } end rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index f11ec11221..06396e9b8e 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.packages.connection_config" provider = p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index 9b3f7f40d9..af231116a3 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,19 +22,68 @@ end scope = "cc.resource_pool.connection_config" provider = p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end - # optional passthrough for extra storage-cli flags +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + +# optional 'custom' field passthrough for extra storage-cli flags +unless options.empty? begin custom = p("#{scope}.custom", {}) if custom.respond_to?(:each) @@ -42,7 +92,6 @@ else rescue # ignore if property not defined end - options = cli_cfg_with_default_timeout(options, 'storage_cli') end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/packages/azure-storage-cli/README.md b/packages/azure-storage-cli/README.md deleted file mode 100644 index 7b55338945..0000000000 --- a/packages/azure-storage-cli/README.md +++ /dev/null @@ -1,9 +0,0 @@ -azure-storage-cli-package -============ -bosh-azure-storage-cli, a command line interface for Azure Storage, is used for BOSH deployments instead of outdated fog-azure tools. - -The file can be downloaded from the following location: - -| Filename | Download URL | -|----------------------------------|-----------------------------------------------------------------------------------------------------------------| -| bosh-azure-storage-cli-0.0.186.tar.gz | [https://github.com/cloudfoundry/bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli/archive/refs/tags/v0.0.186.tar.gz) | diff --git a/packages/azure-storage-cli/packaging b/packages/azure-storage-cli/packaging deleted file mode 100644 index f996419de3..0000000000 --- a/packages/azure-storage-cli/packaging +++ /dev/null @@ -1,5 +0,0 @@ -set -e - -mkdir -p ${BOSH_INSTALL_TARGET}/bin -mv azure-storage-cli/azure-storage-cli-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli -chmod +x ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli diff --git a/packages/azure-storage-cli/spec b/packages/azure-storage-cli/spec deleted file mode 100644 index ec87110de6..0000000000 --- a/packages/azure-storage-cli/spec +++ /dev/null @@ -1,4 +0,0 @@ ---- -name: azure-storage-cli -files: - - azure-storage-cli/azure-storage-cli-linux-amd64 \ No newline at end of file diff --git a/packages/storage-cli/README.md b/packages/storage-cli/README.md new file mode 100644 index 0000000000..395377a584 --- /dev/null +++ b/packages/storage-cli/README.md @@ -0,0 +1,9 @@ +storage-cli-package +============ +storage-cli, a command line interface for interacting with storage provider like;Azure, Aliyun, Google, AWS and WebDav. It is used for BOSH deployments instead of outdated rubyfog tools. + +The file can be downloaded from the following location: + +| Filename | Download URL | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------| +| storage-cli/storage-cli-dev.1.0.0-linux-amd64 | [https://github.com/sap-contributions/storage-cli](https://github.com/sap-contributions/storage-cli/archive/refs/tags/v1.0.0.tar.gz)| diff --git a/packages/storage-cli/packaging b/packages/storage-cli/packaging new file mode 100644 index 0000000000..7d7198bfdf --- /dev/null +++ b/packages/storage-cli/packaging @@ -0,0 +1,6 @@ +set -e + +storage_cli_version="dev.1.0.0" +mkdir -p ${BOSH_INSTALL_TARGET}/bin +mv storage-cli/storage-cli-${storage_cli_version}-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/storage-cli +chmod +x ${BOSH_INSTALL_TARGET}/bin/storage-cli diff --git a/packages/storage-cli/spec b/packages/storage-cli/spec new file mode 100644 index 0000000000..a3441cbbdf --- /dev/null +++ b/packages/storage-cli/spec @@ -0,0 +1,4 @@ +--- +name: storage-cli +files: + - storage-cli/storage-cli-dev.1.0.0-linux-amd64 \ No newline at end of file From 96999413b5817efd13443c6cd9a1a67d5448be62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 16 Dec 2025 12:54:56 +0100 Subject: [PATCH 2/6] fix: bucket_name property added for Google provider --- .../templates/storage_cli_config_buildpacks.json.erb | 3 ++- .../templates/storage_cli_config_droplets.json.erb | 1 + .../templates/storage_cli_config_packages.json.erb | 1 + .../templates/storage_cli_config_resource_pool.json.erb | 1 + .../templates/storage_cli_config_buildpacks.json.erb | 1 + .../templates/storage_cli_config_droplets.json.erb | 1 + .../templates/storage_cli_config_packages.json.erb | 1 + .../templates/storage_cli_config_resource_pool.json.erb | 1 + .../templates/storage_cli_config_buildpacks.json.erb | 1 + .../templates/storage_cli_config_droplets.json.erb | 1 + .../templates/storage_cli_config_packages.json.erb | 1 + .../templates/storage_cli_config_resource_pool.json.erb | 1 + .../templates/storage_cli_config_buildpacks.json.erb | 1 + .../templates/storage_cli_config_droplets.json.erb | 1 + .../templates/storage_cli_config_packages.json.erb | 1 + .../templates/storage_cli_config_resource_pool.json.erb | 1 + 16 files changed, 17 insertions(+), 1 deletion(-) diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb index e5c9c9b41a..94301d8cf5 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb @@ -30,7 +30,7 @@ if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - options["container_name"] = l.p("#{scope}.azure_storage_access_key") + options["account_key"] = l.p("#{scope}.azure_storage_access_key") add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') @@ -40,6 +40,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb index edaf922129..c4b2d86f4c 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb @@ -40,6 +40,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb index 5d210faf28..b3c3b21b76 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb @@ -40,6 +40,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb index 0730a3e855..988cc9d140 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb @@ -40,6 +40,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index 3872194bb2..b3c8dc4d21 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index 362857ce03..b0f24f01c5 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index a958dc06a8..87ef7b5c6b 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index af231116a3..e0bf322618 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index e0c3b11c0c..97ac0d13bf 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index 5d07cfeda8..bbb30dcc4d 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index 06396e9b8e..01bc4406e5 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index af231116a3..e0bf322618 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 1d05a9f96b..9f26a48d6a 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index 362857ce03..b0f24f01c5 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index 06396e9b8e..01bc4406e5 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index af231116a3..e0bf322618 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -38,6 +38,7 @@ if provider == "Google" options["provider"] = provider options["credentials_source"] = "static" options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) end From 5f11ceabbddf473b883db131a1cc66bc8b8035f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 16 Dec 2025 16:39:24 +0100 Subject: [PATCH 3/6] test: unit tests are added --- .../storage_cli_config_jsons_spec.rb | 287 ++++++++++++++++-- .../storage_cli_config_jsons_spec.rb | 280 ++++++++++++++--- .../storage_cli_config_jsons_spec.rb | 283 ++++++++++++++--- .../storage_cli_config_jsons_spec.rb | 280 ++++++++++++++--- 4 files changed, 973 insertions(+), 157 deletions(-) diff --git a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb index 96bbe225e9..e2215dc0e5 100644 --- a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb +++ b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb @@ -14,46 +14,64 @@ module Bosh module Template module Test - RSpec.describe 'storage-cli JSON templates (cc_deployment_updater)' do + RSpec.describe 'storage-cli JSON templates' do let(:release_path) { File.join(File.dirname(__FILE__), '../..') } let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cc_deployment_updater') } - let(:link_props) do + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - let(:cc_link) do - Bosh::Template::Test::Link.new( - name: 'cloud_controller_internal', - properties: link_props - ) - end - - let(:links) { [cc_link] } - let(:props) { {} } + describe 'Unsupported provider' do + let(:link_props) {props_for_provider("Unsupported")} + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end end - TEMPLATES.each do |scope, (template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe "When Provider AzureRM" do + let(:link_props) {props_for_provider("AzureRM")} + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } - context "when provider is AzureRM for #{scope}" do - before do - link_props['cc'][scope.to_s]['blobstore_provider'] = 'AzureRM' - end + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do set(link_props, keypath, { @@ -63,7 +81,6 @@ def set(hash, path, value) 'container_name' => 'cont', 'put_timeout_in_seconds' => '' }) - json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( 'provider' => 'AzureRM', @@ -82,20 +99,228 @@ def set(hash, path, value) 'container_name' => 'cont', 'put_timeout_in_seconds' => '7' }) - json = YAML.safe_load(template.render(props, consumes: links)) expect(json['put_timeout_in_seconds']).to eq('7') end end + end + end + + describe "When Provider AWS" do + let(:link_props) {props_for_provider("AWS")} + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } - context "when provider is non-Azure for #{scope}" do - it 'renders {}' do + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end + + it 'renders with optionals' do + set(link_props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end + end + end + end + + describe "When Provider Google" do + let(:link_props) {props_for_provider("Google")} + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(link_props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) end end end end + + describe "When Provider aliyun" do + let(:link_props) {props_for_provider("aliyun")} + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end + end + end + end + + describe "When Provider webdav" do + let(:link_props) {props_for_provider("webdav")} + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"} + ) + end + + + it 'renders with optionals' do + set(link_props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + "retry_attempts" => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"}, + 'secret' => 'secret', + "retry_attempts" => '4' + ) + end + end + end + end end end end diff --git a/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb index cb388bb101..58803ed56a 100644 --- a/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb @@ -19,64 +19,260 @@ module Test let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cloud_controller_clock') } let(:links) { {} } - let(:props) do + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - TEMPLATES.each_value do |(template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'Unsupported provider' do + let(:props) {props_for_provider("Unsupported")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end + end + + describe "When Provider AzureRM" do + let(:props) {props_for_provider("AzureRM")} + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + end + end + end + + describe "When Provider AWS" do + let(:props) {props_for_provider("AWS")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) + end + end + + describe "When Provider Google" do + let(:props) {props_for_provider("Google")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end end + end + end - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') + describe "When Provider aliyun" do + let(:props) {props_for_provider("aliyun")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end end + end + end + + describe "When Provider webdav" do + let(:props) {props_for_provider("webdav")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"} + ) + end - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + "retry_attempts" => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"}, + 'secret' => 'secret', + "retry_attempts" => '4' + ) + end end end end + end end end diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb index 7759ba71d9..56dc8caf43 100644 --- a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'rspec' +require 'yaml'# frozen_string_literal: true + require 'rspec' require 'yaml' require 'bosh/template/test' @@ -19,64 +22,260 @@ module Test let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cloud_controller_ng') } let(:links) { {} } - let(:props) do + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - TEMPLATES.each_value do |(template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'Unsupported provider' do + let(:props) {props_for_provider("Unsupported")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end + end + + describe "When Provider AzureRM" do + let(:props) {props_for_provider("AzureRM")} + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + end + end + end + + describe "When Provider AWS" do + let(:props) {props_for_provider("AWS")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) + end + end + + describe "When Provider Google" do + let(:props) {props_for_provider("Google")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end end + end + end + + describe "When Provider aliyun" do + let(:props) {props_for_provider("aliyun")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end end + end + end + + describe "When Provider webdav" do + let(:props) {props_for_provider("webdav")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"} + ) + end - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + "retry_attempts" => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"}, + 'secret' => 'secret', + "retry_attempts" => '4' + ) + end end end end + end end end diff --git a/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb index 7ba58af697..a465da3fc7 100644 --- a/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb @@ -19,64 +19,260 @@ module Test let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cloud_controller_worker') } let(:links) { {} } - let(:props) do + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - TEMPLATES.each_value do |(template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'Unsupported provider' do + let(:props) {props_for_provider("Unsupported")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end + end + + describe "When Provider AzureRM" do + let(:props) {props_for_provider("AzureRM")} + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + end + end + end + + describe "When Provider AWS" do + let(:props) {props_for_provider("AWS")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) + end + end + + describe "When Provider Google" do + let(:props) {props_for_provider("Google")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end end + end + end - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') + describe "When Provider aliyun" do + let(:props) {props_for_provider("aliyun")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end end + end + end + + describe "When Provider webdav" do + let(:props) {props_for_provider("webdav")} + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"} + ) + end - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + "retry_attempts" => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => {"cert"=>"some_cert"}, + 'secret' => 'secret', + "retry_attempts" => '4' + ) + end end end end + end end end From 35fe566ea17f883c3e7c4f37f3bb37cfbe9e2293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 16 Dec 2025 16:43:52 +0100 Subject: [PATCH 4/6] feat: custom field removed from generated json - Since these json's also consumed by storage-cli and this custom field not exist at storage-cli config struct. Even though go lang did not complain about extra fields, we should not add. --- .../templates/storage_cli_config_buildpacks.json.erb | 11 ----------- .../templates/storage_cli_config_droplets.json.erb | 11 ----------- .../templates/storage_cli_config_packages.json.erb | 11 ----------- .../storage_cli_config_resource_pool.json.erb | 11 ----------- .../templates/storage_cli_config_buildpacks.json.erb | 11 ----------- .../templates/storage_cli_config_droplets.json.erb | 11 ----------- .../templates/storage_cli_config_packages.json.erb | 11 ----------- .../storage_cli_config_resource_pool.json.erb | 11 ----------- .../templates/storage_cli_config_buildpacks.json.erb | 11 ----------- .../templates/storage_cli_config_droplets.json.erb | 11 ----------- .../templates/storage_cli_config_packages.json.erb | 11 ----------- .../storage_cli_config_resource_pool.json.erb | 11 ----------- .../templates/storage_cli_config_buildpacks.json.erb | 11 ----------- .../templates/storage_cli_config_droplets.json.erb | 11 ----------- .../templates/storage_cli_config_packages.json.erb | 11 ----------- .../storage_cli_config_resource_pool.json.erb | 11 ----------- 16 files changed, 176 deletions(-) diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb index 94301d8cf5..c3725a8146 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb @@ -85,16 +85,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb index c4b2d86f4c..d9a40ba37b 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb @@ -85,16 +85,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb index b3c3b21b76..fb946996d4 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb @@ -85,16 +85,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb index 988cc9d140..b955475662 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb @@ -85,16 +85,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index b3c8dc4d21..8d788ad12f 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index b0f24f01c5..02976bf06f 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index 87ef7b5c6b..66a3a81402 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index e0bf322618..d99170731f 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index 97ac0d13bf..99e61ff5a1 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index bbb30dcc4d..c4f57094ec 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index 01bc4406e5..764a3d7089 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index e0bf322618..d99170731f 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 9f26a48d6a..c8d14535b8 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index b0f24f01c5..02976bf06f 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index 01bc4406e5..764a3d7089 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index e0bf322618..d99170731f 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -83,16 +83,5 @@ if provider == "webdav" end end -# optional 'custom' field passthrough for extra storage-cli flags -unless options.empty? - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end -end -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file From fbfc692915d14a44104a857d86ada415840030fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 16 Dec 2025 16:56:08 +0100 Subject: [PATCH 5/6] style: formating is run --- .../storage_cli_config_jsons_spec.rb | 149 ++++++++--------- .../storage_cli_config_jsons_spec.rb | 150 ++++++++--------- .../storage_cli_config_jsons_spec.rb | 154 +++++++++--------- .../storage_cli_config_jsons_spec.rb | 150 ++++++++--------- 4 files changed, 308 insertions(+), 295 deletions(-) diff --git a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb index e2215dc0e5..994aa86fc2 100644 --- a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb +++ b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb @@ -37,7 +37,7 @@ def props_for_provider(provider) end describe 'Unsupported provider' do - let(:link_props) {props_for_provider("Unsupported")} + let(:link_props) { props_for_provider('Unsupported') } let(:cc_link) do Bosh::Template::Test::Link.new( name: 'cloud_controller_internal', @@ -47,9 +47,10 @@ def props_for_provider(provider) let(:links) { [cc_link] } let(:props) { {} } - TEMPLATES.each_value do |(template_path, keypath)| + TEMPLATES.each_value do |(template_path, _keypath)| describe template_path do let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to eq({}) @@ -58,8 +59,8 @@ def props_for_provider(provider) end end - describe "When Provider AzureRM" do - let(:link_props) {props_for_provider("AzureRM")} + describe 'When Provider AzureRM' do + let(:link_props) { props_for_provider('AzureRM') } let(:cc_link) do Bosh::Template::Test::Link.new( name: 'cloud_controller_internal', @@ -106,8 +107,8 @@ def props_for_provider(provider) end end - describe "When Provider AWS" do - let(:link_props) {props_for_provider("AWS")} + describe 'When Provider AWS' do + let(:link_props) { props_for_provider('AWS') } let(:cc_link) do Bosh::Template::Test::Link.new( name: 'cloud_controller_internal', @@ -131,53 +132,53 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'credentials_source' => 'static', - 'secret_access_key' => 'secret', - 'region' => 'us-east1' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' ) end it 'renders with optionals' do set(link_props, keypath, { - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'aws_access_key_id' => 'key', - 'aws_secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'encryption' => 'some-encryption', - 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', - 'multipart_upload' => 'true' - }) + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'server_side_encryption' => 'some-encryption', - 'sse_kms_key_id' => 'id', - 'multipart_upload' => 'true' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' ) end end end end - describe "When Provider Google" do - let(:link_props) {props_for_provider("Google")} + describe 'When Provider Google' do + let(:link_props) { props_for_provider('Google') } let(:cc_link) do Bosh::Template::Test::Link.new( name: 'cloud_controller_internal', @@ -186,6 +187,7 @@ def props_for_provider(provider) end let(:links) { [cc_link] } let(:props) { {} } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -198,22 +200,22 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'json_key' => '{}', - 'credentials_source' => 'static' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' ) end it 'renders with optionals' do set(link_props, keypath, { - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'google_json_key_string' => '{}', - 'storage_class' => 'STANDARD', - 'encryption_key' => 'key' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' - }) + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( @@ -229,8 +231,8 @@ def props_for_provider(provider) end end - describe "When Provider aliyun" do - let(:link_props) {props_for_provider("aliyun")} + describe 'When Provider aliyun' do + let(:link_props) { props_for_provider('aliyun') } let(:cc_link) do Bosh::Template::Test::Link.new( name: 'cloud_controller_internal', @@ -239,6 +241,7 @@ def props_for_provider(provider) end let(:links) { [cc_link] } let(:props) { {} } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -253,19 +256,19 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'aliyun', - 'access_key_id' => 'key', - 'access_key_secret' => 'secret', - 'endpoint' => 'aliyun.com', - 'bucket_name' => 'bucket' + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' ) end end end end - describe "When Provider webdav" do - let(:link_props) {props_for_provider("webdav")} + describe 'When Provider webdav' do + let(:link_props) { props_for_provider('webdav') } let(:cc_link) do Bosh::Template::Test::Link.new( name: 'cloud_controller_internal', @@ -274,6 +277,7 @@ def props_for_provider(provider) end let(:links) { [cc_link] } let(:props) { {} } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -288,15 +292,14 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"} + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } ) end - it 'renders with optionals' do set(link_props, keypath, { 'provider' => 'webdav', @@ -305,22 +308,22 @@ def props_for_provider(provider) 'public_endpoint' => 'webdav.com', 'ca_cert' => 'some_cert', 'secret' => 'secret', - "retry_attempts" => '4' + 'retry_attempts' => '4' }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"}, - 'secret' => 'secret', - "retry_attempts" => '4' + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' ) end end end - end + end end end end diff --git a/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb index 58803ed56a..716cefa687 100644 --- a/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb @@ -38,10 +38,12 @@ def props_for_provider(provider) end describe 'Unsupported provider' do - let(:props) {props_for_provider("Unsupported")} - TEMPLATES.each_value do |(template_path, keypath)| + let(:props) { props_for_provider('Unsupported') } + + TEMPLATES.each_value do |(template_path, _keypath)| describe template_path do let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to eq({}) @@ -50,8 +52,8 @@ def props_for_provider(provider) end end - describe "When Provider AzureRM" do - let(:props) {props_for_provider("AzureRM")} + describe 'When Provider AzureRM' do + let(:props) { props_for_provider('AzureRM') } TEMPLATES.each_value do |(template_path, keypath)| describe template_path do @@ -90,8 +92,9 @@ def props_for_provider(provider) end end - describe "When Provider AWS" do - let(:props) {props_for_provider("AWS")} + describe 'When Provider AWS' do + let(:props) { props_for_provider('AWS') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -106,53 +109,54 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'credentials_source' => 'static', - 'secret_access_key' => 'secret', - 'region' => 'us-east1' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' ) end it 'renders with optionals' do set(props, keypath, { - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'aws_access_key_id' => 'key', - 'aws_secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'encryption' => 'some-encryption', - 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', - 'multipart_upload' => 'true' - }) + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'server_side_encryption' => 'some-encryption', - 'sse_kms_key_id' => 'id', - 'multipart_upload' => 'true' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' ) end end end end - describe "When Provider Google" do - let(:props) {props_for_provider("Google")} + describe 'When Provider Google' do + let(:props) { props_for_provider('Google') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -165,22 +169,22 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'json_key' => '{}', - 'credentials_source' => 'static' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' ) end it 'renders with optionals' do set(props, keypath, { - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'google_json_key_string' => '{}', - 'storage_class' => 'STANDARD', - 'encryption_key' => 'key' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' - }) + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( @@ -196,8 +200,9 @@ def props_for_provider(provider) end end - describe "When Provider aliyun" do - let(:props) {props_for_provider("aliyun")} + describe 'When Provider aliyun' do + let(:props) { props_for_provider('aliyun') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -212,19 +217,20 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'aliyun', - 'access_key_id' => 'key', - 'access_key_secret' => 'secret', - 'endpoint' => 'aliyun.com', - 'bucket_name' => 'bucket' + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' ) end end end end - describe "When Provider webdav" do - let(:props) {props_for_provider("webdav")} + describe 'When Provider webdav' do + let(:props) { props_for_provider('webdav') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -239,15 +245,14 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"} + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } ) end - it 'renders with optionals' do set(props, keypath, { 'provider' => 'webdav', @@ -256,23 +261,22 @@ def props_for_provider(provider) 'public_endpoint' => 'webdav.com', 'ca_cert' => 'some_cert', 'secret' => 'secret', - "retry_attempts" => '4' + 'retry_attempts' => '4' }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"}, - 'secret' => 'secret', - "retry_attempts" => '4' + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' ) end end end end - end end end diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb index 56dc8caf43..0c4cdca6f7 100644 --- a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -1,10 +1,8 @@ # frozen_string_literal: true require 'rspec' -require 'yaml'# frozen_string_literal: true +require 'yaml' # frozen_string_literal: true -require 'rspec' -require 'yaml' require 'bosh/template/test' TEMPLATES = { @@ -41,10 +39,12 @@ def props_for_provider(provider) end describe 'Unsupported provider' do - let(:props) {props_for_provider("Unsupported")} - TEMPLATES.each_value do |(template_path, keypath)| + let(:props) { props_for_provider('Unsupported') } + + TEMPLATES.each_value do |(template_path, _keypath)| describe template_path do let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to eq({}) @@ -53,8 +53,8 @@ def props_for_provider(provider) end end - describe "When Provider AzureRM" do - let(:props) {props_for_provider("AzureRM")} + describe 'When Provider AzureRM' do + let(:props) { props_for_provider('AzureRM') } TEMPLATES.each_value do |(template_path, keypath)| describe template_path do @@ -93,8 +93,9 @@ def props_for_provider(provider) end end - describe "When Provider AWS" do - let(:props) {props_for_provider("AWS")} + describe 'When Provider AWS' do + let(:props) { props_for_provider('AWS') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -109,53 +110,54 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'credentials_source' => 'static', - 'secret_access_key' => 'secret', - 'region' => 'us-east1' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' ) end it 'renders with optionals' do set(props, keypath, { - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'aws_access_key_id' => 'key', - 'aws_secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'encryption' => 'some-encryption', - 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', - 'multipart_upload' => 'true' - }) + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'server_side_encryption' => 'some-encryption', - 'sse_kms_key_id' => 'id', - 'multipart_upload' => 'true' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' ) end end end end - describe "When Provider Google" do - let(:props) {props_for_provider("Google")} + describe 'When Provider Google' do + let(:props) { props_for_provider('Google') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -168,22 +170,22 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'json_key' => '{}', - 'credentials_source' => 'static' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' ) end it 'renders with optionals' do set(props, keypath, { - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'google_json_key_string' => '{}', - 'storage_class' => 'STANDARD', - 'encryption_key' => 'key' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' - }) + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( @@ -199,8 +201,9 @@ def props_for_provider(provider) end end - describe "When Provider aliyun" do - let(:props) {props_for_provider("aliyun")} + describe 'When Provider aliyun' do + let(:props) { props_for_provider('aliyun') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -215,19 +218,20 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'aliyun', - 'access_key_id' => 'key', - 'access_key_secret' => 'secret', - 'endpoint' => 'aliyun.com', - 'bucket_name' => 'bucket' + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' ) end end end end - describe "When Provider webdav" do - let(:props) {props_for_provider("webdav")} + describe 'When Provider webdav' do + let(:props) { props_for_provider('webdav') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -242,15 +246,14 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"} + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } ) end - it 'renders with optionals' do set(props, keypath, { 'provider' => 'webdav', @@ -259,23 +262,22 @@ def props_for_provider(provider) 'public_endpoint' => 'webdav.com', 'ca_cert' => 'some_cert', 'secret' => 'secret', - "retry_attempts" => '4' + 'retry_attempts' => '4' }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"}, - 'secret' => 'secret', - "retry_attempts" => '4' + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' ) end end end end - end end end diff --git a/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb index a465da3fc7..32fd304000 100644 --- a/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb @@ -38,10 +38,12 @@ def props_for_provider(provider) end describe 'Unsupported provider' do - let(:props) {props_for_provider("Unsupported")} - TEMPLATES.each_value do |(template_path, keypath)| + let(:props) { props_for_provider('Unsupported') } + + TEMPLATES.each_value do |(template_path, _keypath)| describe template_path do let(:template) { job.template(template_path) } + it 'renders empty JSON for unsupported provider' do json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to eq({}) @@ -50,8 +52,8 @@ def props_for_provider(provider) end end - describe "When Provider AzureRM" do - let(:props) {props_for_provider("AzureRM")} + describe 'When Provider AzureRM' do + let(:props) { props_for_provider('AzureRM') } TEMPLATES.each_value do |(template_path, keypath)| describe template_path do @@ -90,8 +92,9 @@ def props_for_provider(provider) end end - describe "When Provider AWS" do - let(:props) {props_for_provider("AWS")} + describe 'When Provider AWS' do + let(:props) { props_for_provider('AWS') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -106,53 +109,54 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'credentials_source' => 'static', - 'secret_access_key' => 'secret', - 'region' => 'us-east1' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' ) end it 'renders with optionals' do set(props, keypath, { - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'aws_access_key_id' => 'key', - 'aws_secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'encryption' => 'some-encryption', - 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', - 'multipart_upload' => 'true' - }) + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'AWS', - 'bucket_name' => 'bucket', - 'access_key_id' => 'key', - 'secret_access_key' => 'secret', - 'region' => 'us-east1', - 'host' => 'localhost', - 'ssl_verify_peer' => 'verfiy', - 'use_ssl' => 'true', - 'singnature_version' => 'v4', - 'server_side_encryption' => 'some-encryption', - 'sse_kms_key_id' => 'id', - 'multipart_upload' => 'true' + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' ) end end end end - describe "When Provider Google" do - let(:props) {props_for_provider("Google")} + describe 'When Provider Google' do + let(:props) { props_for_provider('Google') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -165,22 +169,22 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'json_key' => '{}', - 'credentials_source' => 'static' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' ) end it 'renders with optionals' do set(props, keypath, { - 'provider' => 'Google', - 'bucket_name' => 'bucket', - 'google_json_key_string' => '{}', - 'storage_class' => 'STANDARD', - 'encryption_key' => 'key' + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' - }) + }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( @@ -196,8 +200,9 @@ def props_for_provider(provider) end end - describe "When Provider aliyun" do - let(:props) {props_for_provider("aliyun")} + describe 'When Provider aliyun' do + let(:props) { props_for_provider('aliyun') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -212,19 +217,20 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'aliyun', - 'access_key_id' => 'key', - 'access_key_secret' => 'secret', - 'endpoint' => 'aliyun.com', - 'bucket_name' => 'bucket' + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' ) end end end end - describe "When Provider webdav" do - let(:props) {props_for_provider("webdav")} + describe 'When Provider webdav' do + let(:props) { props_for_provider('webdav') } + TEMPLATES.each_value do |(template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -239,15 +245,14 @@ def props_for_provider(provider) }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"} + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } ) end - it 'renders with optionals' do set(props, keypath, { 'provider' => 'webdav', @@ -256,23 +261,22 @@ def props_for_provider(provider) 'public_endpoint' => 'webdav.com', 'ca_cert' => 'some_cert', 'secret' => 'secret', - "retry_attempts" => '4' + 'retry_attempts' => '4' }) json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( - 'provider' => 'webdav', - 'user' => 'user', - 'password' => 'secret', - 'endpoint' => 'webdav.com', - 'tls' => {"cert"=>"some_cert"}, - 'secret' => 'secret', - "retry_attempts" => '4' + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' ) end end end end - end end end From d1cce59c22e32ca7c074e203fb2e32abcad21822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 19 Dec 2025 12:59:07 +0100 Subject: [PATCH 6/6] feat: first version of storage-cli configured - storage-cli/storage-cli-0.0.1-linux-amd64 uploaded into capi-release-blobs bucket --- config/blobs.yml | 7 ++++--- packages/storage-cli/README.md | 2 +- packages/storage-cli/packaging | 2 +- packages/storage-cli/spec | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/blobs.yml b/config/blobs.yml index 990108d7ec..74f768378e 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -94,9 +94,10 @@ postgres/postgresql-11.22.tar.gz: size: 26826810 object_id: d1f8d34c-b438-44e7-7672-5daea8a6da66 sha: sha256:6445a4e1533c1e8bb616d4a3784bdc4c0226b541f6f0c8d996d9f27d581d49c3 -storage-cli/storage-cli-dev.1.0.0-linux-amd64: - size: 59856924 - sha: sha256:08b5e97f7b1f59254df7d41741106c920a2ce65055f6fd90ddcdcccba7f20c32 +storage-cli/storage-cli-0.0.1-linux-amd64: + size: 59959431 + object_id: 0bd00d1a-88a5-48e3-5908-6b0cc90d5f88 + sha: sha256:302c54413358f07bd1a239aa27c0752167c21dc869ce8a9501e994969be49eea valkey/7.2.11.tar.gz: size: 3441067 object_id: ac3c2823-80c0-41e9-78e4-e42014aad04a diff --git a/packages/storage-cli/README.md b/packages/storage-cli/README.md index 395377a584..07b7cdb3db 100644 --- a/packages/storage-cli/README.md +++ b/packages/storage-cli/README.md @@ -6,4 +6,4 @@ The file can be downloaded from the following location: | Filename | Download URL | |----------------------------------|-----------------------------------------------------------------------------------------------------------------| -| storage-cli/storage-cli-dev.1.0.0-linux-amd64 | [https://github.com/sap-contributions/storage-cli](https://github.com/sap-contributions/storage-cli/archive/refs/tags/v1.0.0.tar.gz)| +| storage-cli/storage-cli-0.0.1-linux-amd64 | [https://github.com/cloudfoundry/storage-cli](https://github.com/cloudfoundry/storage-cli/releases/download/v0.0.1/storage-cli-0.0.1-linux-amd64)| \ No newline at end of file diff --git a/packages/storage-cli/packaging b/packages/storage-cli/packaging index 7d7198bfdf..d4c581ba62 100644 --- a/packages/storage-cli/packaging +++ b/packages/storage-cli/packaging @@ -1,6 +1,6 @@ set -e -storage_cli_version="dev.1.0.0" +storage_cli_version="0.0.1" mkdir -p ${BOSH_INSTALL_TARGET}/bin mv storage-cli/storage-cli-${storage_cli_version}-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/storage-cli chmod +x ${BOSH_INSTALL_TARGET}/bin/storage-cli diff --git a/packages/storage-cli/spec b/packages/storage-cli/spec index a3441cbbdf..0b3239a145 100644 --- a/packages/storage-cli/spec +++ b/packages/storage-cli/spec @@ -1,4 +1,4 @@ --- name: storage-cli files: - - storage-cli/storage-cli-dev.1.0.0-linux-amd64 \ No newline at end of file + - storage-cli/storage-cli-0.0.1-linux-amd64 \ No newline at end of file