S3

1. Avoid rooting URIs (issues when working with Minio on localhost)
2. Use special version of URI encoder
3. Support continuation tokens in listing bucket objects

.github/workflows/release.yml

@@ -22,10 +22,10 @@ jobs:
           grep -m 2 -B 1000 '^## ' bin/release-history.md | tail -n +3 | head -n -2 > release-notes.md
           cat release-notes.md
 
-      - name: Push to nuget.org
+      - name: 📢 Push to nuget.org
         run: dotnet nuget push bin/*.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate
 
-      - name: Create a release
+      - name: 🥰 Create a release
         uses: softprops/action-gh-release@v1
         if: github.ref == 'refs/heads/master'
         with:
@@ -34,4 +34,4 @@ jobs:
           files: |
             bin/*.nupkg
           body_path: release-notes.md
-          discussion_category_name: announcements
+          # discussion_category_name: announcements

docs/release-history.md

@@ -1,4 +1,15 @@
-## 1.3.0
+## 1.4.0
+
+### New features
+
+- [**Minio**](https://min.io/) is officially supported. Additional factory method added allowing to connect to Minio instances without knowing the internals.
+- `IOPath` has a new member - `Prefix()` allowing to add prefix to an existing path.
+
+### Bugs fixed
+
+- **S3** provider was ignoring `continuationToken` which resulted in returning only the first page in `Ls` operation.
+
+## 1.3.0
 
 ### Bugs fixed
 

src/Stowage.Test/BuiltInIntegrationsTest.cs

@@ -36,6 +36,9 @@ private IFileStorage CreateStorage(string name) {
                 case "S3":
                     storage = Files.Of.AmazonS3(settings.AwsBucket, settings.AwsKey, settings.AwsSecret, settings.AwsRegion);
                     break;
+                case "Minio":
+                    storage = Files.Of.Minio(settings.MinioEndpoint, "stowage", settings.MinioKey, settings.MinioSecret);
+                    break;
                 case "GCP":
                     storage = Files.Of.GoogleCloudStorage(settings.GcpBucket, settings.GcpCred.Base64Decode());
                     break;
@@ -107,6 +110,15 @@ public Task S3(string n, Func<Task> testMethod) {
         }
     }
 
+    [Trait("Category", "Integration")]
+    public class MinioIntegrationTest : BuiltInIntegrationsTest {
+        [Theory]
+        [StorageTestData]
+        public Task Minio(string n, Func<Task> testMethod) {
+            return testMethod();
+        }
+    }
+
     [Trait("Category", "Integration")]
     public class GCPIntegrationTest : BuiltInIntegrationsTest {
         [Theory]

src/Stowage.Test/IOPathTest.cs

@@ -1,107 +1,105 @@
 using Xunit;
 
-namespace Stowage.Test
-{
-   public class IOPathTest
-   {
-      [Theory]
-      [InlineData("/dev/one", new[] { "dev", "one" })]
-      [InlineData("/one/two/three", new[] { "one", "two", "three" })]
-      [InlineData("/three", new[] { "one", "..", "three" })]
-      [InlineData("/dev/one/", new[] { "dev", "one/" })]
-      public void Combine_theory(string expected, string[] parts)
-      {
-         Assert.Equal(expected, IOPath.Combine(parts));
-      }
+namespace Stowage.Test {
+    public class IOPathTest {
+        [Theory]
+        [InlineData("/dev/one", new[] { "dev", "one" })]
+        [InlineData("/one/two/three", new[] { "one", "two", "three" })]
+        [InlineData("/three", new[] { "one", "..", "three" })]
+        [InlineData("/dev/one/", new[] { "dev", "one/" })]
+        public void Combine_theory(string expected, string[] parts) {
+            Assert.Equal(expected, IOPath.Combine(parts));
+        }
 
-      [Theory]
-      [InlineData(new[] { "one", "two" }, "one/two")]
-      [InlineData(new[] { "one", "two" }, "/one/two")]
-      [InlineData(new[] { "one", "two/" }, "one/two/")]
-      [InlineData(new[] { "one", "two/" }, "/one/two/")]
-      public void Split_theory(string[] expected, string input)
-      {
-         Assert.Equal(expected, IOPath.Split(input));
-      }
+        [Theory]
+        [InlineData("file", "container", "/container/file")]
+        [InlineData("file/", "container", "/container/file/")]
+        [InlineData("/", "container", "/container/")]
+        public void Prefix_theory(string path, string prefix, string expected) {
+            IOPath pathPath = path;
+            IOPath prefixPath = prefix;
+            Assert.Equal(expected, pathPath.Prefix(prefixPath).Full);
+        }
 
-      [Theory]
-      [InlineData("dev/..", "/")]
-      [InlineData("dev/../storage", "/storage")]
-      [InlineData("/one", "/one")]
-      [InlineData("/one/", "/one/")]
-      [InlineData("/one/../../../..", "/")]
-      public void Normalize_theory(string path, string expected)
-      {
-         Assert.Equal(expected, IOPath.Normalize(path));
-      }
+        [Theory]
+        [InlineData(new[] { "one", "two" }, "one/two")]
+        [InlineData(new[] { "one", "two" }, "/one/two")]
+        [InlineData(new[] { "one", "two/" }, "one/two/")]
+        [InlineData(new[] { "one", "two/" }, "/one/two/")]
+        public void Split_theory(string[] expected, string input) {
+            Assert.Equal(expected, IOPath.Split(input));
+        }
 
-      [Theory]
-      [InlineData("dev1", "/dev1/")]
-      public void Normalize_trailing_theory(string path, string expected)
-      {
-         Assert.Equal(expected, IOPath.Normalize(path, appendTrailingSlash: true));
-      }
+        [Theory]
+        [InlineData("dev/..", "/")]
+        [InlineData("dev/../storage", "/storage")]
+        [InlineData("/one", "/one")]
+        [InlineData("/one/", "/one/")]
+        [InlineData("/one/../../../..", "/")]
+        public void Normalize_theory(string path, string expected) {
+            Assert.Equal(expected, IOPath.Normalize(path));
+        }
 
-      [Theory]
-      [InlineData("one/two/three", "/one/two/")]
-      [InlineData("one/two", "/one/")]
-      [InlineData("one/../two/three", "/two/")]
-      [InlineData("one/../two/three/four/..", "/two/")]
-      public void Get_parent_theory(string path, string expected)
-      {
-         Assert.Equal(expected, IOPath.GetParent(path));
-      }
+        [Theory]
+        [InlineData("dev1", "/dev1/")]
+        public void Normalize_trailing_theory(string path, string expected) {
+            Assert.Equal(expected, IOPath.Normalize(path, appendTrailingSlash: true));
+        }
 
-      [Theory]
-      [InlineData("/one/two", "/one", "/two")]
-      [InlineData("/one/two/", "/one", "/two/")]
-      [InlineData("/one/two", "one", "/two")]
-      [InlineData("/one/two", "/", "/one/two")]
-      [InlineData("/one/two", "x", "/")]
-      [InlineData("/one/two", "/1/2/3/4", "/")]
-      [InlineData("/one/two", null, "/")]
-      [InlineData(null, null, "/")]
-      [InlineData(null, "/", "/")]
-      public void Relative_theory(string path, string relativeTo, string expected)
-      {
-         Assert.Equal(expected, IOPath.RelativeTo(path, relativeTo));
-      }
+        [Theory]
+        [InlineData("one/two/three", "/one/two/")]
+        [InlineData("one/two", "/one/")]
+        [InlineData("one/../two/three", "/two/")]
+        [InlineData("one/../two/three/four/..", "/two/")]
+        public void Get_parent_theory(string path, string expected) {
+            Assert.Equal(expected, IOPath.GetParent(path));
+        }
 
-      [Theory]
-      [InlineData("/", null, true)]
-      [InlineData(null, "/", true)]
-      [InlineData("/", "", true)]
-      [InlineData("/path1", "path1", true)]
-      public void Compare_theory(string path1, string path2, bool expected)
-      {
-         Assert.Equal(expected, IOPath.Compare(path1, path2));
-      }
+        [Theory]
+        [InlineData("/one/two", "/one", "/two")]
+        [InlineData("/one/two/", "/one", "/two/")]
+        [InlineData("/one/two", "one", "/two")]
+        [InlineData("/one/two", "/", "/one/two")]
+        [InlineData("/one/two", "x", "/")]
+        [InlineData("/one/two", "/1/2/3/4", "/")]
+        [InlineData("/one/two", null, "/")]
+        [InlineData(null, null, "/")]
+        [InlineData(null, "/", "/")]
+        public void Relative_theory(string path, string relativeTo, string expected) {
+            Assert.Equal(expected, IOPath.RelativeTo(path, relativeTo));
+        }
 
-      [Theory]
-      [InlineData("/one/", "/one/")]
-      [InlineData("/one", "/one/")]
-      [InlineData("one/", "/one/")]
-      public void WTS_theory(string input, string expected)
-      {
-         Assert.Equal(expected, new IOPath(input).WTS);
-      }
+        [Theory]
+        [InlineData("/", null, true)]
+        [InlineData(null, "/", true)]
+        [InlineData("/", "", true)]
+        [InlineData("/path1", "path1", true)]
+        public void Compare_theory(string path1, string path2, bool expected) {
+            Assert.Equal(expected, IOPath.Compare(path1, path2));
+        }
 
-      [Theory]
-      [InlineData("/one/", "one/")]
-      [InlineData("/one", "one")]
-      [InlineData("one/", "one/")]
-      public void NLS_theory(string input, string expected)
-      {
-         Assert.Equal(expected, new IOPath(input).NLS);
-      }
+        [Theory]
+        [InlineData("/one/", "/one/")]
+        [InlineData("/one", "/one/")]
+        [InlineData("one/", "/one/")]
+        public void WTS_theory(string input, string expected) {
+            Assert.Equal(expected, new IOPath(input).WTS);
+        }
 
-      [Theory]
-      [InlineData("/one/", "one/")]
-      [InlineData("/one", "one/")]
-      [InlineData("one/", "one/")]
-      public void NLWTS_theory(string input, string expected)
-      {
-         Assert.Equal(expected, new IOPath(input).NLWTS);
-      }
-   }
-}
+        [Theory]
+        [InlineData("/one/", "one/")]
+        [InlineData("/one", "one")]
+        [InlineData("one/", "one/")]
+        public void NLS_theory(string input, string expected) {
+            Assert.Equal(expected, new IOPath(input).NLS);
+        }
+
+        [Theory]
+        [InlineData("/one/", "one/")]
+        [InlineData("/one", "one/")]
+        [InlineData("one/", "one/")]
+        public void NLWTS_theory(string input, string expected) {
+            Assert.Equal(expected, new IOPath(input).NLWTS);
+        }
+    }
+}

src/Stowage.Test/ITestSettings.cs

@@ -1,29 +1,33 @@
 using System;
 
-namespace Stowage.Test
-{
-   public interface ITestSettings
-   {
-      string AzureStorageAccount { get; }
+namespace Stowage.Test {
+    public interface ITestSettings {
+        string AzureStorageAccount { get; }
 
-      string AzureStorageKey { get; }
+        string AzureStorageKey { get; }
 
-      string AzureContainerName { get; }
+        string AzureContainerName { get; }
 
-      string AwsBucket { get; }
+        string AwsBucket { get; }
 
-      string AwsKey { get; }
+        string AwsKey { get; }
 
-      string AwsSecret { get; }
+        string AwsSecret { get; }
 
-      string AwsRegion { get; }
+        string AwsRegion { get; }
 
-      string GcpBucket { get; }
+        Uri MinioEndpoint { get; }
 
-      string GcpCred { get; }
+        string MinioKey { get; }
 
-      Uri DatabricksBaseUri { get; }
+        string MinioSecret { get; }
 
-      string DatabricksToken { get; }
-   }
-}
+        string GcpBucket { get; }
+
+        string GcpCred { get; }
+
+        Uri DatabricksBaseUri { get; }
+
+        string DatabricksToken { get; }
+    }
+}

src/Stowage/Extensions.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Stowage {
+    static class Extensions {
+        public static string GetAbsolutePathUnencoded(this Uri uri) {
+            string result = uri.ToString();
+
+            // remove protocol
+            int i = result.IndexOf("://");
+            if(i >= 0)
+                result = result.Substring(i + 3);
+
+            // remove host and port
+            i = result.IndexOf('/');
+            if(i >= 0)
+                result = result.Substring(i);
+
+            // remove query string
+            i = result.IndexOf('?');
+            if(i >= 0)
+                result = result.Substring(0, i);
+
+            return result;
+        }
+    }
+}

src/Stowage/Files.cs

@@ -80,6 +80,33 @@ public static IFileStorage AmazonS3(this IFilesFactory _, string accessKeyId, st
                new S3AuthHandler(accessKeyId, secretAccessKey, region));
         }
 
+        /// <summary>
+        /// Minio storage is a S3-compatible storage.
+        /// Read more at https://min.io/docs/minio/linux/index.html
+        /// </summary>
+        /// <param name="_"></param>
+        /// <param name="endpoint">Minio endpoint, for example http://localhost:9000</param>
+        /// <param name="bucketName">Minio bucket name</param>
+        /// <param name="accessKeyId">Access key you can get from Minio's console "Access keys" tab.</param>
+        /// <param name="secretAccessKey">Secret key for the access key above</param>
+        /// <returns></returns>
+        public static IFileStorage Minio(this IFilesFactory _,
+            Uri endpoint,
+            string bucketName,
+            string accessKeyId,
+            string secretAccessKey) {
+
+            // create endpoint URI using bucketName as a subdomain and add to endpoint parameter
+            //var fullEndpoint = new Uri($"{endpoint.Scheme}://{bucketName}.{endpoint.Authority}");
+
+            // bucket name should be included as a part of the path in Minio
+            var bucketEndpoint = new Uri(endpoint, bucketName + "/");
+
+            return new AwsS3FileStorage(
+               bucketEndpoint,
+               new S3AuthHandler(accessKeyId, secretAccessKey, ""));
+        }
+
         public static IFileStorage DigitalOceanSpaces(this IFilesFactory _, string region, string accessKeyId, string secretAccessKey) {
             return new AwsS3FileStorage(
                new Uri($"https://{region}.digitaloceanspaces.com"),