diff --git a/config/blobs.yml b/config/blobs.yml index 84d5bb2a6e..74f768378e 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -1,7 +1,3 @@ -azure-storage-cli/azure-storage-cli-linux-amd64: - size: 7143608 - object_id: ceb2994f-cbe8-4695-413c-33785a0b6322 - sha: sha256:246944046f7e2f919965466055e957c30aeecf13012059c624d12ab04f48f822 expat/expat-2.5.0.tar.bz2: size: 569205 object_id: 970ccd16-75ac-4c55-5280-c00c4aa8f6cc @@ -98,6 +94,10 @@ postgres/postgresql-11.22.tar.gz: size: 26826810 object_id: d1f8d34c-b438-44e7-7672-5daea8a6da66 sha: sha256:6445a4e1533c1e8bb616d4a3784bdc4c0226b541f6f0c8d996d9f27d581d49c3 +storage-cli/storage-cli-0.0.1-linux-amd64: + size: 59959431 + object_id: 0bd00d1a-88a5-48e3-5908-6b0cc90d5f88 + sha: sha256:302c54413358f07bd1a239aa27c0752167c21dc869ce8a9501e994969be49eea valkey/7.2.11.tar.gz: size: 3441067 object_id: ac3c2823-80c0-41e9-78e4-e42014aad04a diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb index 36ba5f755a..c3725a8146 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpacks bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,28 +24,66 @@ l = link("cloud_controller_internal") scope = "cc.buildpacks.connection_config" provider = l.p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]=l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb index 363a7011f6..d9a40ba37b 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,28 +24,66 @@ l = link("cloud_controller_internal") scope = "cc.droplets.connection_config" provider = l.p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = l.p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]=l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb index 4de210731c..fb946996d4 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,28 +24,66 @@ l = link("cloud_controller_internal") scope = "cc.packages.connection_config" provider = l.p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]= l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb index bbf336fcfb..b955475662 100644 --- a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -23,28 +24,66 @@ l = link("cloud_controller_internal") scope = "cc.resource_pool.connection_config" provider = l.p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = l.p("#{scope}.azure_storage_account_name") options["container_name"] = l.p("#{scope}.container_name") - add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) - add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = l.p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = l.p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = l.p("#{scope}.google_json_key_string") + options["bucket_name"] = l.p("#{scope}.bucket_name") + add_optional(options, "storage_class", l.p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", l.p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = l.p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = l.p("#{scope}.aws_access_key_id") + options["secret_access_key"] = l.p("#{scope}.aws_secret_access_key") + options["region"]=l.p("#{scope}.region") + add_optional(options, "host", l.p("#{scope}.host", nil)) + add_optional(options, "port", l.p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", l.p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", l.p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", l.p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", l.p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", l.p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", l.p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = l.p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = l.p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = l.p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = l.p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = l.p("#{scope}.username") + options["password"] = l.p("#{scope}.password") + options["endpoint"] = l.p("#{scope}.public_endpoint") + add_optional(options, "secret", l.p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", l.p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=l.p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index 5b8a7d3134..b046c7223f 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -31,7 +31,7 @@ templates: storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json packages: - - azure-storage-cli + - storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index 400ccf3d01..8d788ad12f 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpack bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.buildpacks.connection_config" provider = p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index 60e46c9f30..02976bf06f 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.droplets.connection_config" provider = p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index f11ec11221..66a3a81402 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.packages.connection_config" provider = p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index 9b3f7f40d9..d99170731f 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.resource_pool.connection_config" provider = p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index e44390f239..35cdde188e 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -70,7 +70,7 @@ templates: cloud_controller_local_worker_override.yml.erb: config/cloud_controller_local_worker_override.yml packages: - - azure-storage-cli + - storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index 400ccf3d01..99e61ff5a1 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpacs bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.buildpacks.connection_config" provider = p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index 60e46c9f30..c4f57094ec 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.droplets.connection_config" provider = p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index f11ec11221..764a3d7089 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.packages.connection_config" provider = p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index 9b3f7f40d9..d99170731f 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.resource_pool.connection_config" provider = p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 21f5c555f0..c99806be0c 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -41,7 +41,7 @@ templates: storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json packages: - - azure-storage-cli + - storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 400ccf3d01..c8d14535b8 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for buildpacks bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.buildpacks.connection_config" provider = p("cc.buildpacks.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index 60e46c9f30..02976bf06f 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for droplets bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.droplets.connection_config" provider = p("cc.droplets.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index f11ec11221..764a3d7089 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for packages bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.packages.connection_config" provider = p("cc.packages.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index 9b3f7f40d9..d99170731f 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -1,3 +1,4 @@ +<%# Generate json configuration for resource_pool bucket depending on provider; Azure, Google, AWS, Ali, WebDAV %> <% require "json" @@ -13,7 +14,7 @@ def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds end # helper: add key only when value is present -def add(h, key, val) +def add_optional(h, key, val) return if val.nil? return if val.respond_to?(:empty?) && val.empty? h[key] = val @@ -21,28 +22,66 @@ end scope = "cc.resource_pool.connection_config" provider = p("cc.resource_pool.blobstore_provider", nil) +options = {} -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} +if provider == "AzureRM" options["provider"] = provider options["account_name"] = p("#{scope}.azure_storage_account_name") options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end + options["account_key"] = p("#{scope}.azure_storage_access_key") + add_optional(options, "environment", p("#{scope}.environment", "AzureCloud")) + add_optional(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) options = cli_cfg_with_default_timeout(options, 'storage_cli') end + +if provider == "Google" + options["provider"] = provider + options["credentials_source"] = "static" + options["json_key"] = p("#{scope}.google_json_key_string") + options["bucket_name"] = p("#{scope}.bucket_name") + add_optional(options, "storage_class", p("#{scope}.storage_class", nil)) + add_optional(options, "encryption_key", p("#{scope}.encryption_key", nil)) +end + +if provider == "AWS" + options["provider"] = provider + options["bucket_name"] = p("#{scope}.bucket_name") + options["credentials_source"] = "static" + options["access_key_id"] = p("#{scope}.aws_access_key_id") + options["secret_access_key"] = p("#{scope}.aws_secret_access_key") + options["region"]=p("#{scope}.region") + add_optional(options, "host", p("#{scope}.host", nil)) + add_optional(options, "port", p("#{scope}.port", nil)) + add_optional(options, "ssl_verify_peer", p("#{scope}.ssl_verify_peer", nil)) + add_optional(options, "use_ssl", p("#{scope}.use_ssl", nil)) + add_optional(options, "singnature_version", p("#{scope}.singnature_version", nil)) + add_optional(options, "server_side_encryption", p("#{scope}.encryption", nil)) + add_optional(options, "sse_kms_key_id", p("#{scope}.x-amz-server-side-encryption-aws-kms-key-id", nil)) + add_optional(options, "multipart_upload", p("#{scope}.multipart_upload", nil)) +end + +if provider == "aliyun" + options["provider"] = provider + options["access_key_id"] = p("#{scope}.aliyun_accesskey_id") + options["access_key_secret"] = p("#{scope}.aliyun_accesskey_secret") + options["endpoint"] = p("#{scope}.aliyun_oss_endpoint") + options["bucket_name"] = p("#{scope}.aliyun_oss_bucket") +end + +if provider == "webdav" + options["provider"] = provider + options["user"] = p("#{scope}.username") + options["password"] = p("#{scope}.password") + options["endpoint"] = p("#{scope}.public_endpoint") + add_optional(options, "secret", p("#{scope}.secret", nil)) + add_optional(options, "retry_attempts", p("#{scope}.retry_attempts", nil)) + + # TLS nested object with a Cert inside + ca_cert=p("#{scope}.ca_cert",nil) + unless ca_cert.empty? + options["tls"]={"cert"=>ca_cert} + end +end + -%> <%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/packages/azure-storage-cli/README.md b/packages/azure-storage-cli/README.md deleted file mode 100644 index 7b55338945..0000000000 --- a/packages/azure-storage-cli/README.md +++ /dev/null @@ -1,9 +0,0 @@ -azure-storage-cli-package -============ -bosh-azure-storage-cli, a command line interface for Azure Storage, is used for BOSH deployments instead of outdated fog-azure tools. - -The file can be downloaded from the following location: - -| Filename | Download URL | -|----------------------------------|-----------------------------------------------------------------------------------------------------------------| -| bosh-azure-storage-cli-0.0.186.tar.gz | [https://github.com/cloudfoundry/bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli/archive/refs/tags/v0.0.186.tar.gz) | diff --git a/packages/azure-storage-cli/packaging b/packages/azure-storage-cli/packaging deleted file mode 100644 index f996419de3..0000000000 --- a/packages/azure-storage-cli/packaging +++ /dev/null @@ -1,5 +0,0 @@ -set -e - -mkdir -p ${BOSH_INSTALL_TARGET}/bin -mv azure-storage-cli/azure-storage-cli-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli -chmod +x ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli diff --git a/packages/azure-storage-cli/spec b/packages/azure-storage-cli/spec deleted file mode 100644 index ec87110de6..0000000000 --- a/packages/azure-storage-cli/spec +++ /dev/null @@ -1,4 +0,0 @@ ---- -name: azure-storage-cli -files: - - azure-storage-cli/azure-storage-cli-linux-amd64 \ No newline at end of file diff --git a/packages/storage-cli/README.md b/packages/storage-cli/README.md new file mode 100644 index 0000000000..07b7cdb3db --- /dev/null +++ b/packages/storage-cli/README.md @@ -0,0 +1,9 @@ +storage-cli-package +============ +storage-cli, a command line interface for interacting with storage provider like;Azure, Aliyun, Google, AWS and WebDav. It is used for BOSH deployments instead of outdated rubyfog tools. + +The file can be downloaded from the following location: + +| Filename | Download URL | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------| +| storage-cli/storage-cli-0.0.1-linux-amd64 | [https://github.com/cloudfoundry/storage-cli](https://github.com/cloudfoundry/storage-cli/releases/download/v0.0.1/storage-cli-0.0.1-linux-amd64)| \ No newline at end of file diff --git a/packages/storage-cli/packaging b/packages/storage-cli/packaging new file mode 100644 index 0000000000..d4c581ba62 --- /dev/null +++ b/packages/storage-cli/packaging @@ -0,0 +1,6 @@ +set -e + +storage_cli_version="0.0.1" +mkdir -p ${BOSH_INSTALL_TARGET}/bin +mv storage-cli/storage-cli-${storage_cli_version}-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/storage-cli +chmod +x ${BOSH_INSTALL_TARGET}/bin/storage-cli diff --git a/packages/storage-cli/spec b/packages/storage-cli/spec new file mode 100644 index 0000000000..0b3239a145 --- /dev/null +++ b/packages/storage-cli/spec @@ -0,0 +1,4 @@ +--- +name: storage-cli +files: + - storage-cli/storage-cli-0.0.1-linux-amd64 \ No newline at end of file diff --git a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb index 96bbe225e9..994aa86fc2 100644 --- a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb +++ b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb @@ -14,46 +14,65 @@ module Bosh module Template module Test - RSpec.describe 'storage-cli JSON templates (cc_deployment_updater)' do + RSpec.describe 'storage-cli JSON templates' do let(:release_path) { File.join(File.dirname(__FILE__), '../..') } let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cc_deployment_updater') } - let(:link_props) do + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - let(:cc_link) do - Bosh::Template::Test::Link.new( - name: 'cloud_controller_internal', - properties: link_props - ) - end + describe 'Unsupported provider' do + let(:link_props) { props_for_provider('Unsupported') } + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } - let(:links) { [cc_link] } - let(:props) { {} } + TEMPLATES.each_value do |(template_path, _keypath)| + describe template_path do + let(:template) { job.template(template_path) } - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end end - TEMPLATES.each do |scope, (template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'When Provider AzureRM' do + let(:link_props) { props_for_provider('AzureRM') } + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } - context "when provider is AzureRM for #{scope}" do - before do - link_props['cc'][scope.to_s]['blobstore_provider'] = 'AzureRM' - end + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do set(link_props, keypath, { @@ -63,7 +82,6 @@ def set(hash, path, value) 'container_name' => 'cont', 'put_timeout_in_seconds' => '' }) - json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to include( 'provider' => 'AzureRM', @@ -82,16 +100,226 @@ def set(hash, path, value) 'container_name' => 'cont', 'put_timeout_in_seconds' => '7' }) - json = YAML.safe_load(template.render(props, consumes: links)) expect(json['put_timeout_in_seconds']).to eq('7') end end + end + end - context "when provider is non-Azure for #{scope}" do - it 'renders {}' do + describe 'When Provider AWS' do + let(:link_props) { props_for_provider('AWS') } + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end + + it 'renders with optionals' do + set(link_props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end + end + end + end + + describe 'When Provider Google' do + let(:link_props) { props_for_provider('Google') } + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(link_props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end + end + end + end + + describe 'When Provider aliyun' do + let(:link_props) { props_for_provider('aliyun') } + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end + end + end + end + + describe 'When Provider webdav' do + let(:link_props) { props_for_provider('webdav') } + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + let(:links) { [cc_link] } + let(:props) { {} } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(link_props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } + ) + end + + it 'renders with optionals' do + set(link_props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + 'retry_attempts' => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' + ) end end end diff --git a/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb index cb388bb101..716cefa687 100644 --- a/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_clock/storage_cli_config_jsons_spec.rb @@ -19,61 +19,261 @@ module Test let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cloud_controller_clock') } let(:links) { {} } - let(:props) do + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - TEMPLATES.each_value do |(template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'Unsupported provider' do + let(:props) { props_for_provider('Unsupported') } - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + TEMPLATES.each_value do |(template_path, _keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) + end + end + + describe 'When Provider AzureRM' do + let(:props) { props_for_provider('AzureRM') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end end + end + end + + describe 'When Provider AWS' do + let(:props) { props_for_provider('AWS') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end end + end + end + + describe 'When Provider Google' do + let(:props) { props_for_provider('Google') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end + end + end + end + + describe 'When Provider aliyun' do + let(:props) { props_for_provider('aliyun') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end + end + end + end + + describe 'When Provider webdav' do + let(:props) { props_for_provider('webdav') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } + ) + end - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + 'retry_attempts' => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' + ) + end end end end diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb index 7759ba71d9..0c4cdca6f7 100644 --- a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true require 'rspec' -require 'yaml' +require 'yaml' # frozen_string_literal: true + require 'bosh/template/test' TEMPLATES = { @@ -19,61 +20,261 @@ module Test let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cloud_controller_ng') } let(:links) { {} } - let(:props) do + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - TEMPLATES.each_value do |(template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'Unsupported provider' do + let(:props) { props_for_provider('Unsupported') } + + TEMPLATES.each_value do |(template_path, _keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end + end + + describe 'When Provider AzureRM' do + let(:props) { props_for_provider('AzureRM') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + end + end + end + + describe 'When Provider AWS' do + let(:props) { props_for_provider('AWS') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) + end + end + + describe 'When Provider Google' do + let(:props) { props_for_provider('Google') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end end + end + end + + describe 'When Provider aliyun' do + let(:props) { props_for_provider('aliyun') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end end + end + end + + describe 'When Provider webdav' do + let(:props) { props_for_provider('webdav') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } + ) + end - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + 'retry_attempts' => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' + ) + end end end end diff --git a/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb index 7ba58af697..32fd304000 100644 --- a/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_worker/storage_cli_config_jsons_spec.rb @@ -19,61 +19,261 @@ module Test let(:release) { ReleaseDir.new(release_path) } let(:job) { release.job('cloud_controller_worker') } let(:links) { {} } - let(:props) do + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + def props_for_provider(provider) { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => provider }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => provider } } } end - TEMPLATES.each_value do |(template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + describe 'Unsupported provider' do + let(:props) { props_for_provider('Unsupported') } - def set(hash, path, value) - cursor = hash - path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } - cursor[path.last] = value + TEMPLATES.each_value do |(template_path, _keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders empty JSON for unsupported provider' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) + end + end + + describe 'When Provider AzureRM' do + let(:props) { props_for_provider('AzureRM') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end end + end + end + + describe 'When Provider AWS' do + let(:props) { props_for_provider('AWS') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'credentials_source' => 'static', + 'secret_access_key' => 'secret', + 'region' => 'us-east1' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'aws_access_key_id' => 'key', + 'aws_secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'encryption' => 'some-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id' => 'id', + 'multipart_upload' => 'true' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AWS', + 'bucket_name' => 'bucket', + 'access_key_id' => 'key', + 'secret_access_key' => 'secret', + 'region' => 'us-east1', + 'host' => 'localhost', + 'ssl_verify_peer' => 'verfiy', + 'use_ssl' => 'true', + 'singnature_version' => 'v4', + 'server_side_encryption' => 'some-encryption', + 'sse_kms_key_id' => 'id', + 'multipart_upload' => 'true' + ) + end end + end + end + + describe 'When Provider Google' do + let(:props) { props_for_provider('Google') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static' + ) + end + + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'google_json_key_string' => '{}', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'Google', + 'bucket_name' => 'bucket', + 'json_key' => '{}', + 'credentials_source' => 'static', + 'storage_class' => 'STANDARD', + 'encryption_key' => 'key' + ) + end + end + end + end + + describe 'When Provider aliyun' do + let(:props) { props_for_provider('aliyun') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'aliyun', + 'aliyun_accesskey_id' => 'key', + 'aliyun_accesskey_secret' => 'secret', + 'aliyun_oss_endpoint' => 'aliyun.com', + 'aliyun_oss_bucket' => 'bucket' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'aliyun', + 'access_key_id' => 'key', + 'access_key_secret' => 'secret', + 'endpoint' => 'aliyun.com', + 'bucket_name' => 'bucket' + ) + end + end + end + end + + describe 'When Provider webdav' do + let(:props) { props_for_provider('webdav') } + + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' + it 'renders with valid config' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' } + ) + end - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + it 'renders with optionals' do + set(props, keypath, { + 'provider' => 'webdav', + 'username' => 'user', + 'password' => 'secret', + 'public_endpoint' => 'webdav.com', + 'ca_cert' => 'some_cert', + 'secret' => 'secret', + 'retry_attempts' => '4' + }) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'webdav', + 'user' => 'user', + 'password' => 'secret', + 'endpoint' => 'webdav.com', + 'tls' => { 'cert' => 'some_cert' }, + 'secret' => 'secret', + 'retry_attempts' => '4' + ) + end end end end