Add IThrottle and do code clean-up

joelverhagen · Dec 25, 2020 · 4269933 · 4269933
1 parent 626bf87
commit 4269933
Show file tree

Hide file tree

Showing 20 changed files with 256 additions and 114 deletions.
diff --git a/MiniZip.Sandbox/Program.cs b/MiniZip.Sandbox/Program.cs
@@ -10,11 +10,11 @@ namespace Knapcode.MiniZip
 {
     public class Program
     {
-        public static void Main(string[] args)
+        public static async Task Main(string[] args)
         {
             ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
 
-            LargeAsync().GetAwaiter().GetResult();
+            await LargeAsync();
         }
 
         private static async Task SmallAsync()
@@ -112,7 +112,7 @@ private class LoggingHandler : DelegatingHandler
             public long _totalResponseBodyBytes = 0;
             public long _totalContentLength = 0;
             public int _totalRequests = 0;
-            
+
             private static readonly HashSet<string> InterestingHeaders = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
             {
                 "Range",

diff --git a/MiniZip.Test/ConsistencyTest.cs b/MiniZip.Test/ConsistencyTest.cs
@@ -133,7 +133,7 @@ public async Task WithBcl(string path)
             }
         }
 
-        private static IReadOnlyDictionary<string, KnownException> CasesHandledByBcl = new Dictionary<string, KnownException>
+        private static readonly IReadOnlyDictionary<string, KnownException> CasesHandledByBcl = new Dictionary<string, KnownException>
         {
             {
                 "SharpZipLib/ZipFileHandling.FindEntriesInArchiveExtraData/0.zip",
@@ -145,7 +145,7 @@ public async Task WithBcl(string path)
             },
         };
 
-        private static IReadOnlyDictionary<string, KnownException> CasesNotHandledByBcl = new Dictionary<string, KnownException>
+        private static readonly IReadOnlyDictionary<string, KnownException> CasesNotHandledByBcl = new Dictionary<string, KnownException>
         {
         };
 
@@ -223,7 +223,7 @@ public async Task WithSharpZipLib(string path)
             }
         }
 
-        private static IReadOnlyDictionary<string, Action<ZipDirectory>> DifferencesFromSharpZipLib = new Dictionary<string, Action<ZipDirectory>>
+        private static readonly IReadOnlyDictionary<string, Action<ZipDirectory>> DifferencesFromSharpZipLib = new Dictionary<string, Action<ZipDirectory>>
         {
             {
                 "System.IO.Compression/badzipfiles/invaliddate.zip",
@@ -262,7 +262,7 @@ private static void ConvertToForwardSlashes(ZipDirectory zipDirectory)
             }
         }
 
-        private static IReadOnlyDictionary<string, KnownException> CasesHandledBySharpZipLib = new Dictionary<string, KnownException>
+        private static readonly IReadOnlyDictionary<string, KnownException> CasesHandledBySharpZipLib = new Dictionary<string, KnownException>
         {
             {
                 "SharpZipLib/ZipFileHandling.EmbeddedArchive/1.zip",
@@ -278,7 +278,7 @@ private static void ConvertToForwardSlashes(ZipDirectory zipDirectory)
             },
         };
 
-        private static ISet<string> CasesNotHandledBySharpZipLib = new HashSet<string>
+        private static readonly ISet<string> CasesNotHandledBySharpZipLib = new HashSet<string>
         {
             "System.IO.Compression/compat/NullCharFileName_FromUnix.zip",
             "System.IO.Compression/compat/NullCharFileName_FromWindows.zip",

diff --git a/MiniZip.Test/HttpZipProviderTest.cs b/MiniZip.Test/HttpZipProviderTest.cs
@@ -210,7 +210,7 @@ public async Task HandlesChangingETagProperly(ETagBehavior etagBehavior, bool su
                                 "The HTTP response did not have the expected status code HTTP 206 Partial Content. The response was 412 Precondition Failed.",
                                 ex.Message);
                         }
-                        
+
                     }
                 }
             }
@@ -264,7 +264,7 @@ public void Dispose()
 
         private class TestMessageHandler : HttpMessageHandler
         {
-            private Func<HttpRequestMessage, HttpResponseMessage> _getResponse;
+            private readonly Func<HttpRequestMessage, HttpResponseMessage> _getResponse;
 
             public TestMessageHandler(Func<HttpRequestMessage, HttpResponseMessage> getResponse)
             {

diff --git a/MiniZip.Test/MZip/BoundedStreamTest.cs b/MiniZip.Test/MZip/BoundedStreamTest.cs
@@ -199,7 +199,7 @@ public Facts()
             protected void Verify(int startPosition, int endPosition, byte[] actual, int extraBytes)
             {
                 var count = (endPosition - startPosition) + 1;
-                var expected = new byte[count  + extraBytes];
+                var expected = new byte[count + extraBytes];
                 Buffer.BlockCopy(_bytes, startPosition, expected, 0, count);
                 Assert.Equal(expected, actual);
             }

diff --git a/MiniZip.Test/TestSupport/TestUtility.cs b/MiniZip.Test/TestSupport/TestUtility.cs
@@ -16,7 +16,7 @@ namespace Knapcode.MiniZip
 {
     public static class TestUtility
     {
-        private static object TestDataLock = new object();
+        private static readonly object TestDataLock = new object();
         public static readonly string TestDataDirectory = Path.GetFullPath("TestData");
         public const string TestServerDirectory = "TestData";
 

diff --git a/MiniZip.Test/ZipDirectoryReaderTest.cs b/MiniZip.Test/ZipDirectoryReaderTest.cs
@@ -183,7 +183,7 @@ public async Task CanReadFromAChangedStream()
                     TestUtility.VerifyJsonEquals(expected, actual);
                 }
             }
-            
+
             [Theory]
             [InlineData("System.IO.Compression/refzipfiles/fake64.zip", 770, 942)]
             [InlineData("System.IO.Compression/refzipfiles/normal.zip", 2670582, 2671162)]

diff --git a/MiniZip.sln b/MiniZip.sln
@@ -3,10 +3,10 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.28922.388
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MiniZip.Test", "MiniZip.Test\MiniZip.Test.csproj", "{C76E2794-8411-4124-9F13-14263C7FACEC}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MiniZip.Sandbox", "MiniZip.Sandbox\MiniZip.Sandbox.csproj", "{1A043D05-699B-4268-9B27-0E2411DE8D34}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MiniZip.Test", "MiniZip.Test\MiniZip.Test.csproj", "{C76E2794-8411-4124-9F13-14263C7FACEC}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MiniZip", "MiniZip\MiniZip.csproj", "{64925A3F-B787-407F-8482-965381B1C571}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{09524EDA-0A1E-466C-AC04-1FE7AD9D9E11}"

diff --git a/MiniZip/BufferedRangeStream/BufferedRangeStream.cs b/MiniZip/BufferedRangeStream/BufferedRangeStream.cs
@@ -59,7 +59,7 @@ public override async Task<int> ReadAsync(byte[] buffer, int offset, int count,
                 return 0;
             }
 
-            var read = (int) Math.Min(_buffer.Length - bufferOffset, count);
+            var read = (int)Math.Min(_buffer.Length - bufferOffset, count);
             _buffer.Position = bufferOffset;
             _buffer.ReadExactly(buffer, offset, read);
 
@@ -95,9 +95,9 @@ private async Task<int> GetBufferOffsetAsync(int count)
                 {
                     readOffset = Position;
                 }
-                
+
                 // Read up until the old position (or up to the end, if this is the first read).
-                var readCount =  (int)(BufferPosition - readOffset);
+                var readCount = (int)(BufferPosition - readOffset);
                 var newBuffer = new byte[readCount];
                 var actualRead = await _rangeReader.ReadAsync(readOffset, newBuffer, 0, readCount);
                 _buffer.Prepend(newBuffer);

diff --git a/MiniZip/BufferedRangeStream/HttpRangeReader.cs b/MiniZip/BufferedRangeStream/HttpRangeReader.cs
@@ -16,6 +16,7 @@ public class HttpRangeReader : IRangeReader
         private readonly long _length;
         private readonly EntityTagHeaderValue _etag;
         private readonly bool _requireContentRange;
+        private readonly IThrottle _httpThrottle;
 
         /// <summary>
         /// Initializes the HTTP range reader. The provided <see cref="HttpClient"/> is not disposed by this instance.
@@ -25,13 +26,28 @@ public class HttpRangeReader : IRangeReader
         /// <param name="length">The length of the request URI, a size in bytes.</param>
         /// <param name="etag">The optional etag header to be used for follow-up requests.</param>
         /// <param name="requireContentRange">Whether or not to require and validate the Content-Range header.</param>
-        public HttpRangeReader(HttpClient httpClient, Uri requestUri, long length, EntityTagHeaderValue etag, bool requireContentRange)
+        /// <param name="httpThrottle">The throttle to use for HTTP operations.</param>
+        public HttpRangeReader(HttpClient httpClient, Uri requestUri, long length, EntityTagHeaderValue etag, bool requireContentRange, IThrottle httpThrottle)
         {
             _httpClient = httpClient;
             _requestUri = requestUri;
             _length = length;
             _etag = etag;
             _requireContentRange = requireContentRange;
+            _httpThrottle = httpThrottle ?? NullThrottle.Instance;
+        }
+
+        /// <summary>
+        /// Initializes the HTTP range reader. The provided <see cref="HttpClient"/> is not disposed by this instance.
+        /// </summary>
+        /// <param name="httpClient">The HTTP client used to make the HTTP requests.</param>
+        /// <param name="requestUri">The URL to request bytes from.</param>
+        /// <param name="length">The length of the request URI, a size in bytes.</param>
+        /// <param name="etag">The optional etag header to be used for follow-up requests.</param>
+        /// <param name="requireContentRange">Whether or not to require and validate the Content-Range header.</param>
+        public HttpRangeReader(HttpClient httpClient, Uri requestUri, long length, EntityTagHeaderValue etag, bool requireContentRange)
+            : this(httpClient, requestUri, length, etag, requireContentRange: true, httpThrottle: null)
+        {
         }
 
         /// <summary>
@@ -58,58 +74,66 @@ public async Task<int> ReadAsync(long srcOffset, byte[] dst, int dstOffset, int
         {
             return await RetryHelper.RetryAsync(async () =>
             {
-                using (var request = new HttpRequestMessage(HttpMethod.Get, _requestUri))
+                await _httpThrottle.WaitAsync();
+                try
                 {
-                    request.Headers.Range = new RangeHeaderValue(srcOffset, (srcOffset + count) - 1);
-
-                    if (_etag != null)
+                    using (var request = new HttpRequestMessage(HttpMethod.Get, _requestUri))
                     {
-                        request.Headers.IfMatch.Add(_etag);
-                    }
+                        request.Headers.Range = new RangeHeaderValue(srcOffset, (srcOffset + count) - 1);
 
-                    using (var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
-                    {
-                        if (response.StatusCode != HttpStatusCode.PartialContent)
+                        if (_etag != null)
                         {
-                            throw new MiniZipHttpStatusCodeException(
-                                string.Format(
-                                    Strings.NonPartialContentHttpResponse,
-                                    (int)response.StatusCode,
-                                    response.ReasonPhrase),
-                                response.StatusCode,
-                                response.ReasonPhrase);
+                            request.Headers.IfMatch.Add(_etag);
                         }
 
-                        if (_requireContentRange || response.Content.Headers.ContentRange != null)
+                        using (var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
                         {
-                            if (response.Content.Headers.ContentRange == null)
+                            if (response.StatusCode != HttpStatusCode.PartialContent)
                             {
-                                throw new MiniZipException(Strings.ContentRangeHeaderNotFound);
+                                throw new MiniZipHttpStatusCodeException(
+                                    string.Format(
+                                        Strings.NonPartialContentHttpResponse,
+                                        (int)response.StatusCode,
+                                        response.ReasonPhrase),
+                                    response.StatusCode,
+                                    response.ReasonPhrase);
                             }
 
-                            if (!response.Content.Headers.ContentRange.HasRange
-                                || response.Content.Headers.ContentRange.Unit != HttpConstants.BytesUnit
-                                || response.Content.Headers.ContentRange.From != srcOffset
-                                || response.Content.Headers.ContentRange.To != (srcOffset + count) - 1)
+                            if (_requireContentRange || response.Content.Headers.ContentRange != null)
                             {
-                                throw new MiniZipException(Strings.InvalidContentRangeHeader);
+                                if (response.Content.Headers.ContentRange == null)
+                                {
+                                    throw new MiniZipException(Strings.ContentRangeHeaderNotFound);
+                                }
+
+                                if (!response.Content.Headers.ContentRange.HasRange
+                                    || response.Content.Headers.ContentRange.Unit != HttpConstants.BytesUnit
+                                    || response.Content.Headers.ContentRange.From != srcOffset
+                                    || response.Content.Headers.ContentRange.To != (srcOffset + count) - 1)
+                                {
+                                    throw new MiniZipException(Strings.InvalidContentRangeHeader);
+                                }
+
+                                if (response.Content.Headers.ContentRange.Length != _length)
+                                {
+                                    throw new MiniZipException(string.Format(
+                                        Strings.LengthOfHttpContentChanged,
+                                        response.Content.Headers.ContentRange.Length,
+                                        _length));
+                                }
                             }
 
-                            if (response.Content.Headers.ContentRange.Length != _length)
+                            using (var stream = await response.Content.ReadAsStreamAsync())
                             {
-                                throw new MiniZipException(string.Format(
-                                    Strings.LengthOfHttpContentChanged,
-                                    response.Content.Headers.ContentRange.Length,
-                                    _length));
+                                return await stream.ReadToEndAsync(dst, dstOffset, count);
                             }
                         }
-
-                        using (var stream = await response.Content.ReadAsStreamAsync())
-                        {
-                            return await stream.ReadToEndAsync(dst, dstOffset, count);
-                        }
                     }
                 }
+                finally
+                {
+                    _httpThrottle.Release();
+                }
             });
         }
     }

diff --git a/MiniZip/BufferedRangeStream/SeekableStreamRangeReader.cs b/MiniZip/BufferedRangeStream/SeekableStreamRangeReader.cs
@@ -28,7 +28,7 @@ public SeekableStreamRangeReader(Func<Task<Stream>> openStreamAsync)
         /// <param name="dstOffset">The offset in the destination buffer.</param>
         /// <param name="count">The maximum number of bytes to read.</param>
         /// <returns>The number of bytes read.</returns>
-        public async virtual Task<int> ReadAsync(long srcOffset, byte[] dst, int dstOffset, int count)
+        public virtual async Task<int> ReadAsync(long srcOffset, byte[] dst, int dstOffset, int count)
         {
             using (var stream = await _openStreamAsync())
             {

diff --git a/MiniZip/BufferedRangeStream/ZipBufferSizeProvider.cs b/MiniZip/BufferedRangeStream/ZipBufferSizeProvider.cs
@@ -60,7 +60,7 @@ public int GetNextBufferSize()
 
             try
             {
-                _nextValue = checked(_nextValue  * _exponent);
+                _nextValue = checked(_nextValue * _exponent);
             }
             catch (OverflowException)
             {
-Original file line number
+Diff line change
@@ Expand Up / @@ -60,7 +60,7 @@ public int GetNextBufferSize() @@
                 try
                 {
-                    _nextValue = checked(_nextValue  * _exponent);
+                    _nextValue = checked(_nextValue * _exponent);
                 }
                 catch (OverflowException)
                 {
@@ Expand Down @@