Merge pull request #298 from pierre-hamy/1.1.4-ibm-sdk_FEAT

Allow using bucket endpoint having virtual-host pattern

Author: gilv

README.md

@@ -175,6 +175,7 @@ The following is the list of the Stocator configuration keys. `<service>` can be
 |fs.cos.`<service>`.iam.api.key | API key  | mandatory | value of `apiKey`
 |fs.cos.`<service>`.iam.service.id  | Service ID | mandatory | Value of `iam_serviceid_crn`. In certain cases you need only value after `:serviceid:`
 |fs.cos.`<service>`.endpoint | COS endpoint | mandatory | Open link from `endpoints` and choose relevant endpoint. This endpoint should go here
+|fs.cos.`<service>`.path.style.access | Path style access | Optional | true (default): Path style access <BR>false: Virtual host style access
 |fs.cos.`<service>`.iam.token.max.retry | IAM token retrieval max retry  | Optional |
 |fs.cos.`<service>`.iam.token.refresh | IAM token refresh time offset in seconds  | Optional |
 
@@ -211,6 +212,7 @@ The following is the list of the configuration keys. `<service>` can be any valu
 |fs.cos.`<service>`.access.key | Access key  | mandatory
 |fs.cos.`<service>`.secret.key  | Secret key | mandatory
 |fs.cos.`<service>`.endpoint | COS endpoint | mandatory
+|fs.cos.`<service>`.path.style.access | Path style access | Optional
 |fs.cos.`<service>`.v2.signer.type |  Signer type  | optional
 
 Example, configure `<service>` as `myCOS`:

src/main/java/com/ibm/stocator/fs/cos/COSAPIClient.java

@@ -41,18 +41,14 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Date;
-import java.util.concurrent.CountDownLatch;
 
 import com.ibm.stocator.fs.cache.MemoryCache;
 import com.ibm.stocator.fs.common.Constants;
 import com.ibm.stocator.fs.common.IStoreClient;
 import com.ibm.stocator.fs.common.StocatorPath;
 import com.ibm.stocator.fs.common.Utils;
 import com.ibm.stocator.fs.common.exception.ConfigurationParseException;
-import com.ibm.stocator.fs.cos.ConfigurationHandler;
-import com.ibm.stocator.fs.cos.OnetimeInitialization;
 import com.ibm.stocator.fs.cos.auth.CustomTokenManager;
-import com.ibm.stocator.fs.cos.COSInputStream;
 import com.ibm.cloud.objectstorage.services.s3.AmazonS3ClientBuilder;
 import com.ibm.cloud.objectstorage.AmazonClientException;
 import com.ibm.cloud.objectstorage.AmazonServiceException;
@@ -150,6 +146,8 @@
 import static com.ibm.stocator.fs.cos.COSConstants.BLOCK_SIZE_COS_PROPERTY;
 import static com.ibm.stocator.fs.cos.COSConstants.COS_BUCKET_PROPERTY;
 import static com.ibm.stocator.fs.cos.COSConstants.ENDPOINT_URL_COS_PROPERTY;
+import static com.ibm.stocator.fs.cos.COSConstants.PATH_STYLE_ACCESS;
+import static com.ibm.stocator.fs.cos.COSConstants.DEFAULT_PATH_STYLE_ACCESS;
 import static com.ibm.stocator.fs.common.Constants.FS_STOCATOR_FMODE_DATA_CLEANUP;
 import static com.ibm.stocator.fs.cos.COSConstants.REGION_COS_PROPERTY;
 import static com.ibm.stocator.fs.cos.COSConstants.V2_SIGNER_TYPE_COS_PROPERTY;
@@ -422,9 +420,13 @@ public void initiate(String scheme) throws IOException, ConfigurationParseExcept
     }
     final String serviceUrl = props.getProperty(ENDPOINT_URL_COS_PROPERTY);
 
+    final boolean pathStyleAccessEnabled = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS,
+            PATH_STYLE_ACCESS, DEFAULT_PATH_STYLE_ACCESS);
+    LOG.debug("Setting path style access to {}", pathStyleAccessEnabled);
+
     AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard()
         .withClientConfiguration(clientConf)
-        .withPathStyleAccessEnabled(true)
+        .withPathStyleAccessEnabled(pathStyleAccessEnabled)
         .withCredentials(credProvider);
 
     if (serviceUrl != null && !serviceUrl.equals(amazonDefaultEndpoint)) {

src/main/java/com/ibm/stocator/fs/cos/COSConstants.java

@@ -67,6 +67,10 @@ public class COSConstants {
   public static final String ENDPOINT_URL = ".endpoint";
   public static final String ENDPOINT_URL_COS_PROPERTY = FS_COS + ENDPOINT_URL;
 
+
+  public static final String PATH_STYLE_ACCESS = ".path.style.access";
+  public static final boolean DEFAULT_PATH_STYLE_ACCESS = true;
+
   public static final String BLOCK_SIZE = ".block.size";
   public static final String BLOCK_SIZE_COS_PROPERTY = FS_COS + BLOCK_SIZE;
 

src/main/java/com/ibm/stocator/fs/cos/ConfigurationHandler.java

@@ -38,6 +38,8 @@
 import static com.ibm.stocator.fs.cos.COSConstants.SECRET_KEY_COS_PROPERTY;
 import static com.ibm.stocator.fs.cos.COSConstants.ENDPOINT_URL;
 import static com.ibm.stocator.fs.cos.COSConstants.ENDPOINT_URL_COS_PROPERTY;
+import static com.ibm.stocator.fs.cos.COSConstants.PATH_STYLE_ACCESS;
+import static com.ibm.stocator.fs.cos.COSConstants.DEFAULT_PATH_STYLE_ACCESS;
 import static com.ibm.stocator.fs.cos.COSConstants.AUTO_BUCKET_CREATE;
 import static com.ibm.stocator.fs.cos.COSConstants.AUTO_BUCKET_CREATE_COS_PROPERTY;
 import static com.ibm.stocator.fs.cos.COSConstants.BLOCK_SIZE;
@@ -106,6 +108,8 @@ public static Properties initialize(URI uri, Configuration conf,
         SECRET_KEY_COS_PROPERTY, false);
     Utils.updateProperty(conf, prefix, altPrefix, ENDPOINT_URL, props,
         ENDPOINT_URL_COS_PROPERTY, false);
+    Utils.updateProperty(conf, prefix, altPrefix, PATH_STYLE_ACCESS, props,
+            String.valueOf(DEFAULT_PATH_STYLE_ACCESS), false);
     Utils.updateProperty(conf, prefix, altPrefix, AUTO_BUCKET_CREATE, props,
         AUTO_BUCKET_CREATE_COS_PROPERTY, false);
     Utils.updateProperty(conf, prefix, altPrefix, V2_SIGNER_TYPE, props,