encoding detector

Author: na1307

EncodingConverter/Converter.cs

@@ -2,7 +2,6 @@
 using System.IO;
 using System.Linq;
 using System.Text;
-using System.Threading.Tasks;
 
 namespace EncodingConverter;
 
@@ -19,7 +18,7 @@ public static async Task ConvertEncodingAsync(IEnumerable<FileInfo> files, Encod
 
             string fullText;
 
-            using (StreamReader sr = new(file.FullName, await getEncodingAsync(file.FullName))) {
+            using (StreamReader sr = new(file.FullName, detectTextEncoding(file.FullName, out _, 0))) {
                 fullText = await sr.ReadToEndAsync();
             }
 
@@ -31,29 +30,95 @@ public static async Task ConvertEncodingAsync(IEnumerable<FileInfo> files, Encod
         }
     }
 
-    /// <summary>
-    /// Determines a text file's encoding by analyzing its byte order mark (BOM).
-    /// Defaults to ASCII when detection of the text file's endianness fails.
-    /// </summary>
-    /// <param name="filename">The text file to analyze.</param>
-    /// <returns>The detected encoding.</returns>
-    private static async Task<Encoding> getEncodingAsync(string filename) {
-        // Read the BOM
-        var bom = new byte[4];
-
-        using (FileStream file = new(filename, FileMode.Open, FileAccess.Read)) {
-            await file.ReadAsync(bom, 0, 4);
+    // From https://stackoverflow.com/questions/1025332/determine-a-strings-encoding-in-c-sharp
+    // Function to detect the encoding for UTF-7, UTF-8/16/32 (bom, no bom, little
+    // & big endian), and local default codepage, and potentially other codepages.
+    // 'taster' = number of bytes to check of the file (to save processing). Higher
+    // value is slower, but more reliable (especially UTF-8 with special characters
+    // later on may appear to be ASCII initially). If taster = 0, then taster
+    // becomes the length of the file (for maximum reliability). 'text' is simply
+    // the string with the discovered encoding applied to the file.
+    private static Encoding detectTextEncoding(string filename, out string text, int taster = 1000) {
+        var b = File.ReadAllBytes(filename);
+
+        //////////////// First check the low hanging fruit by checking if a
+        //////////////// BOM/signature exists (sourced from http://www.unicode.org/faq/utf_bom.html#bom4)
+        if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF) { text = Encoding.GetEncoding("utf-32BE").GetString(b, 4, b.Length - 4); return Encoding.GetEncoding("utf-32BE"); }  // UTF-32, big-endian
+        else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00) { text = Encoding.UTF32.GetString(b, 4, b.Length - 4); return Encoding.UTF32; }    // UTF-32, little-endian
+        else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF) { text = Encoding.BigEndianUnicode.GetString(b, 2, b.Length - 2); return Encoding.BigEndianUnicode; }     // UTF-16, big-endian
+        else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE) { text = Encoding.Unicode.GetString(b, 2, b.Length - 2); return Encoding.Unicode; }              // UTF-16, little-endian
+        else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF) { text = Encoding.UTF8.GetString(b, 3, b.Length - 3); return Encoding.UTF8; } // UTF-8
+        else if (b.Length >= 3 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76) { text = Encoding.UTF7.GetString(b, 3, b.Length - 3); return Encoding.UTF7; } // UTF-7
+
+
+        //////////// If the code reaches here, no BOM/signature was found, so now
+        //////////// we need to 'taste' the file to see if can manually discover
+        //////////// the encoding. A high taster value is desired for UTF-8
+        if (taster == 0 || taster > b.Length) taster = b.Length;    // Taster size can't be bigger than the filesize obviously.
+
+
+        // Some text files are encoded in UTF8, but have no BOM/signature. Hence
+        // the below manually checks for a UTF8 pattern. This code is based off
+        // the top answer at: https://stackoverflow.com/questions/6555015/check-for-invalid-utf8
+        // For our purposes, an unnecessarily strict (and terser/slower)
+        // implementation is shown at: https://stackoverflow.com/questions/1031645/how-to-detect-utf-8-in-plain-c
+        // For the below, false positives should be exceedingly rare (and would
+        // be either slightly malformed UTF-8 (which would suit our purposes
+        // anyway) or 8-bit extended ASCII/UTF-16/32 at a vanishingly long shot).
+        int i = 0;
+        bool utf8 = false;
+        while (i < taster - 4) {
+            if (b[i] <= 0x7F) { i += 1; continue; }     // If all characters are below 0x80, then it is valid UTF8, but UTF8 is not 'required' (and therefore the text is more desirable to be treated as the default codepage of the computer). Hence, there's no "utf8 = true;" code unlike the next three checks.
+            if (b[i] >= 0xC2 && b[i] < 0xE0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0) { i += 2; utf8 = true; continue; }
+            if (b[i] >= 0xE0 && b[i] < 0xF0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0) { i += 3; utf8 = true; continue; }
+            if (b[i] >= 0xF0 && b[i] < 0xF5 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0 && b[i + 3] >= 0x80 && b[i + 3] < 0xC0) { i += 4; utf8 = true; continue; }
+            utf8 = false; break;
+        }
+        if (utf8) {
+            text = Encoding.UTF8.GetString(b);
+            return Encoding.UTF8;
         }
 
-        // Analyze the BOM
-        return bom[0] switch {
-            0x2b when bom[1] == 0x2f && bom[2] == 0x76 => Encoding.UTF7,
-            0xef when bom[1] == 0xbb && bom[2] == 0xbf => Encoding.UTF8,
-            0xff when bom[1] == 0xfe && bom[2] == 0 && bom[3] == 0 => Encoding.UTF32, // UTF-32LE
-            0xff when bom[1] == 0xfe => Encoding.Unicode, // UTF-16LE
-            0xfe when bom[1] == 0xff => Encoding.BigEndianUnicode, // UTF-16BE
-            0 when bom[1] == 0 && bom[2] == 0xfe && bom[3] == 0xff => new UTF32Encoding(true, true), // UTF-32BE
-            _ => Encoding.Default, // We actually have no idea what the encoding is if we reach this point, so
-        };                         // you may wish to return null instead of defaulting to ASCII
+
+        // The next check is a heuristic attempt to detect UTF-16 without a BOM.
+        // We simply look for zeroes in odd or even byte places, and if a certain
+        // threshold is reached, the code is 'probably' UF-16.
+        double threshold = 0.1; // proportion of chars step 2 which must be zeroed to be diagnosed as utf-16. 0.1 = 10%
+        int count = 0;
+        for (int n = 0; n < taster; n += 2) if (b[n] == 0) count++;
+        if (((double)count) / taster > threshold) { text = Encoding.BigEndianUnicode.GetString(b); return Encoding.BigEndianUnicode; }
+        count = 0;
+        for (int n = 1; n < taster; n += 2) if (b[n] == 0) count++;
+        if (((double)count) / taster > threshold) { text = Encoding.Unicode.GetString(b); return Encoding.Unicode; } // (little-endian)
+
+
+        // Finally, a long shot - let's see if we can find "charset=xyz" or
+        // "encoding=xyz" to identify the encoding:
+        for (int n = 0; n < taster - 9; n++) {
+            if (
+                ((b[n + 0] == 'c' || b[n + 0] == 'C') && (b[n + 1] == 'h' || b[n + 1] == 'H') && (b[n + 2] == 'a' || b[n + 2] == 'A') && (b[n + 3] == 'r' || b[n + 3] == 'R') && (b[n + 4] == 's' || b[n + 4] == 'S') && (b[n + 5] == 'e' || b[n + 5] == 'E') && (b[n + 6] == 't' || b[n + 6] == 'T') && (b[n + 7] == '=')) ||
+                ((b[n + 0] == 'e' || b[n + 0] == 'E') && (b[n + 1] == 'n' || b[n + 1] == 'N') && (b[n + 2] == 'c' || b[n + 2] == 'C') && (b[n + 3] == 'o' || b[n + 3] == 'O') && (b[n + 4] == 'd' || b[n + 4] == 'D') && (b[n + 5] == 'i' || b[n + 5] == 'I') && (b[n + 6] == 'n' || b[n + 6] == 'N') && (b[n + 7] == 'g' || b[n + 7] == 'G') && (b[n + 8] == '='))
+                ) {
+                if (b[n + 0] == 'c' || b[n + 0] == 'C') n += 8; else n += 9;
+                if (b[n] == '"' || b[n] == '\'') n++;
+                int oldn = n;
+                while (n < taster && (b[n] == '_' || b[n] == '-' || (b[n] >= '0' && b[n] <= '9') || (b[n] >= 'a' && b[n] <= 'z') || (b[n] >= 'A' && b[n] <= 'Z'))) { n++; }
+                byte[] nb = new byte[n - oldn];
+                Array.Copy(b, oldn, nb, 0, n - oldn);
+                try {
+                    string internalEnc = Encoding.ASCII.GetString(nb);
+                    text = Encoding.GetEncoding(internalEnc).GetString(b);
+                    return Encoding.GetEncoding(internalEnc);
+                } catch { break; }    // If C# doesn't recognize the name of the encoding, break.
+            }
+        }
+
+
+        // If all else fails, the encoding is probably (though certainly not
+        // definitely) the user's local codepage! One might present to the user a
+        // list of alternative encodings as shown here: https://stackoverflow.com/questions/8509339/what-is-the-most-common-encoding-of-each-language
+        // A full list can be found using Encoding.GetEncodings
+        text = Encoding.Default.GetString(b);
+        return Encoding.Default;
     }
 }

EncodingConverter/Dialogs/ChooseEncodingDialog.xaml.cs

@@ -7,12 +7,15 @@ namespace EncodingConverter;
 /// ChooseEncodingDialog.xaml에 대한 상호 작용 논리
 /// </summary>
 public partial class ChooseEncodingDialog {
+    private static readonly string[] encodings = ["System Encoding", "UTF-8 with BOM", "UTF-8 without BOM"];
+    private static readonly UTF8Encoding utf8withBom = new(encoderShouldEmitUTF8Identifier: true);
+    private static readonly UTF8Encoding utf8withoutBom = new(encoderShouldEmitUTF8Identifier: false);
     //private const int utf8CodePage = 65001;
 
     public ChooseEncodingDialog() {
         InitializeComponent();
         //encodingsCombo.ItemsSource = Encoding.GetEncodings().Select(ei => $"{ei.DisplayName} ({ei.CodePage})");
-        encodingsCombo.ItemsSource = new string[] { "System Encoding", "UTF-8 with BOM", "UTF-8 without BOM" };
+        encodingsCombo.ItemsSource = encodings;
         encodingsCombo.SelectedIndex = encodingsCombo.Items.Count - 1;
     }
 
@@ -24,8 +27,8 @@ public Encoding ChosenEncoding {
 
             return encodingsCombo.SelectedIndex switch {
                 0 => Encoding.Default,
-                1 => new UTF8Encoding(encoderShouldEmitUTF8Identifier: true),
-                2 => new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
+                1 => utf8withBom,
+                2 => utf8withoutBom,
                 _ => throw new InvalidOperationException(),
             };
         }

EncodingConverter/source.extension.cs

@@ -9,7 +9,7 @@ internal sealed partial class Vsix
     {
         public const string Id = "EncodingConverter.625f3018-1e5f-4b92-ad9c-08d2534b6021";
         public const string Name = "Encoding Converter";
-        public const string Description = @"An extension that allows you to easily convert the encoding of multiple files.";
+        public const string Description = @"An extension that allows you to easily convert the encoding of multiple files. (System Encoding / UTF-8 with / without BOM)";
         public const string Language = "en-US";
         public const string Version = "1.0.1";
         public const string Author = "Bluehill";

EncodingConverter/source.extension.vsixmanifest

@@ -3,7 +3,7 @@
     <Metadata>
         <Identity Id="EncodingConverter.625f3018-1e5f-4b92-ad9c-08d2534b6021" Version="1.0.1" Language="en-US" Publisher="Bluehill" />
         <DisplayName>Encoding Converter</DisplayName>
-        <Description xml:space="preserve">An extension that allows you to easily convert the encoding of multiple files.</Description>
+        <Description xml:space="preserve">An extension that allows you to easily convert the encoding of multiple files. (System Encoding / UTF-8 with / without BOM)</Description>
         <License>..\LICENSE.txt</License>
         <Icon>Resources\Icon.png</Icon>
         <PreviewImage>Resources\Icon.png</PreviewImage>