#175 - Decode multipart headers as UTF-8.

Author: Tratcher

src/Microsoft.AspNet.WebUtilities/BufferedReadStream.cs

@@ -11,8 +11,8 @@ namespace Microsoft.AspNet.WebUtilities
 {
     internal class BufferedReadStream : Stream
     {
-        private const char CR = '\r';
-        private const char LF = '\n';
+        private const byte CR = (byte)'\r';
+        private const byte LF = (byte)'\n';
 
         private readonly Stream _inner;
         private readonly byte[] _buffer;
@@ -310,8 +310,9 @@ public async Task<bool> EnsureBufferedAsync(int minCount, CancellationToken canc
         public string ReadLine(int lengthLimit)
         {
             CheckDisposed();
-            StringBuilder builder = new StringBuilder();
+            var builder = new MemoryStream(200);
             bool foundCR = false, foundCRLF = false;
+
             while (!foundCRLF && EnsureBuffered())
             {
                 if (builder.Length > lengthLimit)
@@ -321,19 +322,15 @@ public string ReadLine(int lengthLimit)
                 ProcessLineChar(builder, ref foundCR, ref foundCRLF);
             }
 
-            if (foundCRLF)
-            {
-                return builder.ToString(0, builder.Length - 2); // Drop the CRLF
-            }
-            // Stream ended with no CRLF.
-            return builder.ToString();
+            return DecodeLine(builder, foundCRLF);
         }
 
         public async Task<string> ReadLineAsync(int lengthLimit, CancellationToken cancellationToken)
         {
             CheckDisposed();
-            StringBuilder builder = new StringBuilder();
+            var builder = new MemoryStream(200);
             bool foundCR = false, foundCRLF = false;
+
             while (!foundCRLF && await EnsureBufferedAsync(cancellationToken))
             {
                 if (builder.Length > lengthLimit)
@@ -344,25 +341,20 @@ public async Task<string> ReadLineAsync(int lengthLimit, CancellationToken cance
                 ProcessLineChar(builder, ref foundCR, ref foundCRLF);
             }
 
-            if (foundCRLF)
-            {
-                return builder.ToString(0, builder.Length - 2); // Drop the CRLF
-            }
-            // Stream ended with no CRLF.
-            return builder.ToString();
+            return DecodeLine(builder, foundCRLF);
         }
 
-        private void ProcessLineChar(StringBuilder builder, ref bool foundCR, ref bool foundCRLF)
+        private void ProcessLineChar(MemoryStream builder, ref bool foundCR, ref bool foundCRLF)
         {
-            char ch = (char)_buffer[_bufferOffset]; // TODO: Encoding enforcement
-            builder.Append(ch);
+            var b = _buffer[_bufferOffset];
+            builder.WriteByte(b);
             _bufferOffset++;
             _bufferCount--;
-            if (ch == CR)
+            if (b == CR)
             {
                 foundCR = true;
             }
-            else if (ch == LF)
+            else if (b == LF)
             {
                 if (foundCR)
                 {
@@ -375,6 +367,13 @@ private void ProcessLineChar(StringBuilder builder, ref bool foundCR, ref bool f
             }
         }
 
+        private string DecodeLine(MemoryStream builder, bool foundCRLF)
+        {
+            // Drop the final CRLF, if any
+            var length = foundCRLF ? builder.Length - 2 : builder.Length;
+            return Encoding.UTF8.GetString(builder.ToArray(), 0, (int)length);
+        }
+
         private void CheckDisposed()
         {
             if (_disposed)

test/Microsoft.AspNet.WebUtilities.Tests/MultipartReaderTests.cs

@@ -43,6 +43,19 @@ public class MultipartReaderTests
 
 Content of a.txt.
 
+--9051914041544843365972754266--
+";
+        private const string TwoPartBodyWithUnicodeFileName =
+@"--9051914041544843365972754266
+Content-Disposition: form-data; name=""text""
+
+text default
+--9051914041544843365972754266
+Content-Disposition: form-data; name=""file1""; filename=""a色.txt""
+Content-Type: text/plain
+
+Content of a.txt.
+
 --9051914041544843365972754266--
 ";
         private const string ThreePartBody =
@@ -147,6 +160,32 @@ public async Task MutipartReader_ReadTwoPartBody_Success()
             Assert.Null(await reader.ReadNextSectionAsync());
         }
 
+        [Fact]
+        public async Task MutipartReader_ReadTwoPartBodyWithUnicodeFileName_Success()
+        {
+            var stream = MakeStream(TwoPartBodyWithUnicodeFileName);
+            var reader = new MultipartReader(Boundary, stream);
+
+            var section = await reader.ReadNextSectionAsync();
+            Assert.NotNull(section);
+            Assert.Equal(1, section.Headers.Count);
+            Assert.Equal("form-data; name=\"text\"", section.Headers["Content-Disposition"][0]);
+            var buffer = new MemoryStream();
+            await section.Body.CopyToAsync(buffer);
+            Assert.Equal("text default", Encoding.ASCII.GetString(buffer.ToArray()));
+
+            section = await reader.ReadNextSectionAsync();
+            Assert.NotNull(section);
+            Assert.Equal(2, section.Headers.Count);
+            Assert.Equal("form-data; name=\"file1\"; filename=\"a色.txt\"", section.Headers["Content-Disposition"][0]);
+            Assert.Equal("text/plain", section.Headers["Content-Type"][0]);
+            buffer = new MemoryStream();
+            await section.Body.CopyToAsync(buffer);
+            Assert.Equal("Content of a.txt.\r\n", Encoding.ASCII.GetString(buffer.ToArray()));
+
+            Assert.Null(await reader.ReadNextSectionAsync());
+        }
+
         [Fact]
         public async Task MutipartReader_ThreePartBody_Success()
         {
@@ -181,5 +220,77 @@ public async Task MutipartReader_ThreePartBody_Success()
 
             Assert.Null(await reader.ReadNextSectionAsync());
         }
+
+        [Fact]
+        public async Task MutipartReader_ReadInvalidUtf8Header_ReplacementCharacters()
+        {
+            var body1 =
+@"--9051914041544843365972754266
+Content-Disposition: form-data; name=""text"" filename=""a";
+
+            var body2 =
+@".txt""
+
+text default
+--9051914041544843365972754266--
+";
+            var stream = new MemoryStream();
+            var bytes = Encoding.UTF8.GetBytes(body1);
+            stream.Write(bytes, 0, bytes.Length);
+
+            // Write an invalid utf-8 segment in the middle
+            stream.Write(new byte[] { 0xC1, 0x21 }, 0, 2);
+
+            bytes = Encoding.UTF8.GetBytes(body2);
+            stream.Write(bytes, 0, bytes.Length);
+            stream.Seek(0, SeekOrigin.Begin);
+            var reader = new MultipartReader(Boundary, stream);
+
+            var section = await reader.ReadNextSectionAsync();
+            Assert.NotNull(section);
+            Assert.Equal(1, section.Headers.Count);
+            Assert.Equal("form-data; name=\"text\" filename=\"a\uFFFD!.txt\"", section.Headers["Content-Disposition"][0]);
+            var buffer = new MemoryStream();
+            await section.Body.CopyToAsync(buffer);
+            Assert.Equal("text default", Encoding.ASCII.GetString(buffer.ToArray()));
+
+            Assert.Null(await reader.ReadNextSectionAsync());
+        }
+
+        [Fact]
+        public async Task MutipartReader_ReadInvalidUtf8SurrogateHeader_ReplacementCharacters()
+        {
+            var body1 =
+@"--9051914041544843365972754266
+Content-Disposition: form-data; name=""text"" filename=""a";
+
+            var body2 =
+@".txt""
+
+text default
+--9051914041544843365972754266--
+";
+            var stream = new MemoryStream();
+            var bytes = Encoding.UTF8.GetBytes(body1);
+            stream.Write(bytes, 0, bytes.Length);
+
+            // Write an invalid utf-8 segment in the middle
+            stream.Write(new byte[] { 0xED, 0xA0, 85 }, 0, 3);
+
+            bytes = Encoding.UTF8.GetBytes(body2);
+            stream.Write(bytes, 0, bytes.Length);
+            stream.Seek(0, SeekOrigin.Begin);
+            var reader = new MultipartReader(Boundary, stream);
+
+            var section = await reader.ReadNextSectionAsync();
+            Assert.NotNull(section);
+            Assert.Equal(1, section.Headers.Count);
+            Assert.Equal("form-data; name=\"text\" filename=\"a\uFFFDU.txt\"", section.Headers["Content-Disposition"][0]);
+            var buffer = new MemoryStream();
+            await section.Body.CopyToAsync(buffer);
+            Assert.Equal("text default", Encoding.ASCII.GetString(buffer.ToArray()));
+
+            Assert.Null(await reader.ReadNextSectionAsync());
+        }
     }
 }