Storage CLI Client

lib/cloud_controller/blobstore/client_provider.rb

@@ -5,7 +5,6 @@
 require 'cloud_controller/blobstore/webdav/dav_client'
 require 'cloud_controller/blobstore/safe_delete_client'
 require 'cloud_controller/blobstore/storage_cli/storage_cli_client'
-require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client'
 require 'google/apis/errors'
 
 module CloudController
@@ -72,7 +71,7 @@ def provide_webdav(options, directory_key, root_dir)
         end
 
         def provide_storage_cli(options, directory_key, root_dir, resource_type)
-          client = StorageCliClient.build(
+          client = StorageCliClient.new(
             directory_key: directory_key,
             resource_type: resource_type,
             root_dir: root_dir,

lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb

@@ -4,100 +4,80 @@
 require 'fileutils'
 require 'cloud_controller/blobstore/base_client'
 require 'cloud_controller/blobstore/storage_cli/storage_cli_blob'
-
 module CloudController
   module Blobstore
     class StorageCliClient < BaseClient
       attr_reader :root_dir, :min_size, :max_size
 
-      @registry = {}
-
-      class << self
-        attr_reader :registry
-
-        def register(provider, klass)
-          registry[provider.to_s] = klass
-        end
-
-        def build(directory_key:, root_dir:, resource_type: nil, min_size: nil, max_size: nil)
-          raise 'Missing resource_type' if resource_type.nil?
-
-          cfg = fetch_config(resource_type)
-          provider = cfg['provider']
-
-          key        = provider.to_s
-          impl_class = registry[key] || registry[key.downcase] || registry[key.upcase]
-          raise "No storage CLI client registered for provider #{provider}" unless impl_class
-
-          impl_class.new(provider: provider, directory_key: directory_key, root_dir: root_dir, resource_type: resource_type, min_size: min_size, max_size: max_size,
-                         config_path: config_path_for(resource_type))
-        end
-
-        RESOURCE_TYPE_KEYS = {
-          'droplets' => :storage_cli_config_file_droplets,
-          'buildpack_cache' => :storage_cli_config_file_droplets,
-          'buildpacks' => :storage_cli_config_file_buildpacks,
-          'packages' => :storage_cli_config_file_packages,
-          'resource_pool' => :storage_cli_config_file_resource_pool
-        }.freeze
-
-        def fetch_config(resource_type)
-          path = config_path_for(resource_type)
-          validate_config_path!(path)
-
-          json = fetch_json(path)
-          validate_json_object!(json, path)
-          validate_required_keys!(json, path)
-
-          json
-        end
+      RESOURCE_TYPE_KEYS = {
+        'droplets' => :storage_cli_config_file_droplets,
+        'buildpack_cache' => :storage_cli_config_file_droplets,
+        'buildpacks' => :storage_cli_config_file_buildpacks,
+        'packages' => :storage_cli_config_file_packages,
+        'resource_pool' => :storage_cli_config_file_resource_pool
+      }.freeze
+
+      PROVIDER_TO_STORAGE_CLI_STORAGETYPE = {
+        'AzureRM' => 'azurebs',
+        'aliyun' => 'alioss',
+        'AWS' => 's3',
+        'webdav' => 'dav',
+        'Google' => 'gcs'
+      }.freeze
+
+      IMPLEMENTED_PROVIDERS = %w[AzureRM aliyun].freeze
+
+      def initialize(directory_key:, resource_type:, root_dir:, min_size: nil, max_size: nil)
+        raise 'Missing resource_type' if resource_type.nil?
+
+        config_file_path = config_path_for(resource_type)
+        cfg = fetch_config(resource_type)
+        @provider = cfg['provider'].to_s
+        raise BlobstoreError.new("No provider specified in config file: #{File.basename(config_file_path)}") if @provider.empty?
+        raise "Unimplemented provider: #{@provider}, implemented ones are: #{IMPLEMENTED_PROVIDERS.join(', ')}" unless IMPLEMENTED_PROVIDERS.include?(@provider)
 
-        def config_path_for(resource_type)
-          normalized = resource_type.to_s
-          key = RESOURCE_TYPE_KEYS.fetch(normalized) do
-            raise BlobstoreError.new("Unknown resource_type: #{resource_type}")
-          end
-          VCAP::CloudController::Config.config.get(key)
-        end
-
-        def fetch_json(path)
-          Oj.load(File.read(path))
-        rescue Oj::ParseError, EncodingError => e
-          raise BlobstoreError.new("Failed to parse storage-cli JSON at #{path}: #{e.message}")
-        end
+        @cli_path = cli_path
+        @config_file = config_file_path
+        @directory_key = directory_key
+        @resource_type = resource_type.to_s
+        @root_dir = root_dir
+        @min_size = min_size || 0
+        @max_size = max_size
+        @storage_type = PROVIDER_TO_STORAGE_CLI_STORAGETYPE[@provider]
+        logger.info('initialized with:', resource_type: @resource_type, provider: @provider, path: @config_file)
+      end
 
-        def validate_config_path!(path)
-          return if path && File.file?(path) && File.readable?(path)
+      def fetch_config(resource_type)
+        path = config_path_for(resource_type)
+        validate_config_path!(path)
 
-          raise BlobstoreError.new("Storage-cli config file not found or not readable at: #{path.inspect}")
-        end
+        json = fetch_json(path)
+        validate_json_object!(json, path)
+        json
+      end
 
-        def validate_json_object!(json, path)
-          raise BlobstoreError.new("Config at #{path} must be a JSON object") unless json.is_a?(Hash)
+      def config_path_for(resource_type)
+        normalized = resource_type.to_s
+        key = RESOURCE_TYPE_KEYS.fetch(normalized) do
+          raise BlobstoreError.new("Unknown resource_type: #{resource_type}")
         end
+        VCAP::CloudController::Config.config.get(key)
+      end
 
-        def validate_required_keys!(json, path)
-          provider = json['provider'].to_s.strip
-          raise BlobstoreError.new("No provider specified in config file: #{path.inspect}") if provider.empty?
+      def fetch_json(path)
+        Oj.load(File.read(path))
+      rescue Oj::ParseError, EncodingError => e
+        raise BlobstoreError.new("Failed to parse storage-cli JSON at #{path}: #{e.message}")
+      end
 
-          required = %w[account_key account_name container_name environment]
-          missing = required.reject { |k| json.key?(k) && !json[k].to_s.strip.empty? }
-          return if missing.empty?
+      def validate_config_path!(path)
+        return if path && File.file?(path) && File.readable?(path)
 
-          raise BlobstoreError.new("Missing required keys in #{path}: #{missing.join(', ')}")
-        end
+        raise BlobstoreError.new("Storage-cli config file not found or not readable at: #{path.inspect}")
       end
 
-      def initialize(provider:, directory_key:, resource_type:, root_dir:, config_path:, min_size: nil, max_size: nil)
-        @cli_path = cli_path
-        @directory_key = directory_key
-        @resource_type = resource_type.to_s
-        @root_dir = root_dir
-        @min_size = min_size || 0
-        @max_size = max_size
-        @provider = provider
-        @config_file = config_path
-        logger.info('storage_cli_config_selected', resource_type: @resource_type, path: @config_file)
+      def validate_json_object!(json, path)
+        raise BlobstoreError.new("Config at #{path} must be a JSON object") unless json.is_a?(Hash)
       end
 
       def local?
@@ -183,16 +163,16 @@ def files_for(prefix, _ignored_directory_prefixes=[])
       end
 
       def ensure_bucket_exists
-        run_cli('ensure-bucket-exists')
+        run_cli('ensure-storage-exists')
       end
 
       private
 
       def run_cli(command, *args, allow_exit_code_three: false)
-        logger.info("[storage_cli_client] Running storage-cli: #{@cli_path} -c #{@config_file} #{command} #{args.join(' ')}")
+        logger.info("running storage-cli: #{@cli_path} -c #{@config_file} #{command} #{args.join(' ')}")
 
         begin
-          stdout, stderr, status = Open3.capture3(@cli_path, '-c', @config_file, command, *args)
+          stdout, stderr, status = Open3.capture3(@cli_path, '-s', @storage_type, '-c', @config_file, command, *args)
         rescue StandardError => e
           raise BlobstoreError.new(e.inspect)
         end
@@ -224,7 +204,7 @@ def properties(key)
       end
 
       def cli_path
-        raise NotImplementedError
+        ENV['STORAGE_CLI_PATH'] || '/var/vcap/packages/storage-cli/bin/storage-cli'
       end
 
       def build_config(connection_config)

spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb

@@ -146,10 +146,10 @@ module Blobstore
         end
 
         it 'provides a storage-cli client' do
-          allow(StorageCliClient).to receive(:build).and_return(storage_cli_client_mock)
+          allow(StorageCliClient).to receive(:new).and_return(storage_cli_client_mock)
           ClientProvider.provide(options:, directory_key:, root_dir:, resource_type:)
-          expect(StorageCliClient).to have_received(:build).with(directory_key: directory_key, resource_type: resource_type, root_dir: root_dir,
-                                                                 min_size: 100, max_size: 1000)
+          expect(StorageCliClient).to have_received(:new).with(directory_key: directory_key, resource_type: resource_type, root_dir: root_dir,
+                                                               min_size: 100, max_size: 1000)
         end
 
         it 'raises an error if provider is not provided' do