在C＃中确定一个string的编码

有没有什么办法来确定在C＃中的string的编码？

说，我有一个文件名string，但我不知道它是否以Unicode UTF-16或系统默认编码编码，我怎么知道？

Answer 1

查看Utf8Checker它是纯粹的托pipe代码中正是这样做的简单的类。 http://utf8checker.codeplex.com

注意：正如已经指出的那样，“确定编码”仅适用于字节stream。 如果你有一个string，它已经从某个已经知道或者猜测到编码的人那里编码得到了string。

Answer 2

下面的代码具有以下特点：

1. 检测或试图检测UTF-7，UTF-8/16/32（bom，no bom，little＆big endian）
2. 如果没有findUnicode编码，则返回到本地默认代码页。
3. 检测（高概率）缺lessBOM /签名的unicode文件
4. 在文件内searchcharset = xyz和encoding = xyz以帮助确定编码。
5. 为了节省处理，您可以“品尝”文件（可定义的字节数）。
6. 编码和解码的文本文件被返回。
7. 纯粹的字节为基础的解决scheme的效率

正如其他人所说，没有解决scheme可以是完美的（当然，不能很好地区分世界范围内使用的各种8位扩展ASCII编码），但是如果开发者也向用户呈现，我们可以得到“足够好”如下所示的可选编码列表： 每种语言最常见的编码是什么？

编码的完整列表可以使用Encoding.GetEncodings();

 // Function to detect the encoding for UTF-7, UTF-8/16/32 (bom, no bom, little // & big endian), and local default codepage, and potentially other codepages. // 'taster' = number of bytes to check of the file (to save processing). Higher // value is slower, but more reliable (especially UTF-8 with special characters // later on may appear to be ASCII initially). If taster = 0, then taster // becomes the length of the file (for maximum reliability). 'text' is simply // the string with the discovered encoding applied to the file. public Encoding detectTextEncoding(string filename, out String text, int taster = 1000) { byte[] b = File.ReadAllBytes(filename); //////////////// First check the low hanging fruit by checking if a //////////////// BOM/signature exists (sourced from http://www.unicode.org/faq/utf_bom.html#bom4) if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF) { text = Encoding.GetEncoding("utf-32BE").GetString(b, 4, b.Length - 4); return Encoding.GetEncoding("utf-32BE"); } // UTF-32, big-endian else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00) { text = Encoding.UTF32.GetString(b, 4, b.Length - 4); return Encoding.UTF32; } // UTF-32, little-endian else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF) { text = Encoding.BigEndianUnicode.GetString(b, 2, b.Length - 2); return Encoding.BigEndianUnicode; } // UTF-16, big-endian else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE) { text = Encoding.Unicode.GetString(b, 2, b.Length - 2); return Encoding.Unicode; } // UTF-16, little-endian else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF) { text = Encoding.UTF8.GetString(b, 3, b.Length - 3); return Encoding.UTF8; } // UTF-8 else if (b.Length >= 3 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76) { text = Encoding.UTF7.GetString(b,3,b.Length-3); return Encoding.UTF7; } // UTF-7 //////////// If the code reaches here, no BOM/signature was found, so now //////////// we need to 'taste' the file to see if can manually discover //////////// the encoding. A high taster value is desired for UTF-8 if (taster == 0 || taster > b.Length) taster = b.Length; // Taster size can't be bigger than the filesize obviously. // Some text files are encoded in UTF8, but have no BOM/signature. Hence // the below manually checks for a UTF8 pattern. This code is based off // the top answer at: https://stackoverflow.com/questions/6555015/check-for-invalid-utf8 // For our purposes, an unnecessarily strict (and terser/slower) // implementation is shown at: https://stackoverflow.com/questions/1031645/how-to-detect-utf-8-in-plain-c // For the below, false positives should be exceedingly rare (and would // be either slightly malformed UTF-8 (which would suit our purposes // anyway) or 8-bit extended ASCII/UTF-16/32 at a vanishingly long shot). int i = 0; bool utf8 = false; while (i < taster - 4) { if (b[i] <= 0x7F) { i += 1; continue; } // If all characters are below 0x80, then it is valid UTF8, but UTF8 is not 'required' (and therefore the text is more desirable to be treated as the default codepage of the computer). Hence, there's no "utf8 = true;" code unlike the next three checks. if (b[i] >= 0xC2 && b[i] <= 0xDF && b[i + 1] >= 0x80 && b[i + 1] < 0xC0) { i += 2; utf8 = true; continue; } if (b[i] >= 0xE0 && b[i] <= 0xF0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0) { i += 3; utf8 = true; continue; } if (b[i] >= 0xF0 && b[i] <= 0xF4 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0 && b[i + 3] >= 0x80 && b[i + 3] < 0xC0) { i += 4; utf8 = true; continue; } utf8 = false; break; } if (utf8 == true) { text = Encoding.UTF8.GetString(b); return Encoding.UTF8; } // The next check is a heuristic attempt to detect UTF-16 without a BOM. // We simply look for zeroes in odd or even byte places, and if a certain // threshold is reached, the code is 'probably' UF-16. double threshold = 0.1; // proportion of chars step 2 which must be zeroed to be diagnosed as utf-16. 0.1 = 10% int count = 0; for (int n = 0; n < taster; n += 2) if (b[n] == 0) count++; if (((double)count) / taster > threshold) { text = Encoding.BigEndianUnicode.GetString(b); return Encoding.BigEndianUnicode; } count = 0; for (int n = 1; n < taster; n += 2) if (b[n] == 0) count++; if (((double)count) / taster > threshold) { text = Encoding.Unicode.GetString(b); return Encoding.Unicode; } // (little-endian) // Finally, a long shot - let's see if we can find "charset=xyz" or // "encoding=xyz" to identify the encoding: for (int n = 0; n < taster-9; n++) { if ( ((b[n + 0] == 'c' || b[n + 0] == 'C') && (b[n + 1] == 'h' || b[n + 1] == 'H') && (b[n + 2] == 'a' || b[n + 2] == 'A') && (b[n + 3] == 'r' || b[n + 3] == 'R') && (b[n + 4] == 's' || b[n + 4] == 'S') && (b[n + 5] == 'e' || b[n + 5] == 'E') && (b[n + 6] == 't' || b[n + 6] == 'T') && (b[n + 7] == '=')) || ((b[n + 0] == 'e' || b[n + 0] == 'E') && (b[n + 1] == 'n' || b[n + 1] == 'N') && (b[n + 2] == 'c' || b[n + 2] == 'C') && (b[n + 3] == 'o' || b[n + 3] == 'O') && (b[n + 4] == 'd' || b[n + 4] == 'D') && (b[n + 5] == 'i' || b[n + 5] == 'I') && (b[n + 6] == 'n' || b[n + 6] == 'N') && (b[n + 7] == 'g' || b[n + 7] == 'G') && (b[n + 8] == '=')) ) { if (b[n + 0] == 'c' || b[n + 0] == 'C') n += 8; else n += 9; if (b[n] == '"' || b[n] == '\'') n++; int oldn = n; while (n < taster && (b[n] == '_' || b[n] == '-' || (b[n] >= '0' && b[n] <= '9') || (b[n] >= 'a' && b[n] <= 'z') || (b[n] >= 'A' && b[n] <= 'Z'))) { n++; } byte[] nb = new byte[n-oldn]; Array.Copy(b, oldn, nb, 0, n-oldn); try { string internalEnc = Encoding.ASCII.GetString(nb); text = Encoding.GetEncoding(internalEnc).GetString(b); return Encoding.GetEncoding(internalEnc); } catch { break; } // If C# doesn't recognize the name of the encoding, break. } } // If all else fails, the encoding is probably (though certainly not // definitely) the user's local codepage! One might present to the user a // list of alternative encodings as shown here: https://stackoverflow.com/questions/8509339/what-is-the-most-common-encoding-of-each-language // A full list can be found using Encoding.GetEncodings(); text = Encoding.Default.GetString(b); return Encoding.Default; } 

Answer 3

这取决于string“来自”的位置。 .NETstring是Unicode（UTF-16）。 如果你将数据库中的数据读取到字节数组中，唯一的方法可能是不同的。

这个CodeProject文章可能是有趣的： 检测编码为input和输出文本

Jon Skeet的C＃和.NET中的string是.NETstring的绝佳解释。

Answer 4

我知道这有点晚了，但要清楚：

一个string没有真正的编码…在.NET中，一个string是一个char对象的集合。 本质上，如果它是一个string，它已经被解码。

但是，如果您正在读取由字节组成的文件的内容，并希望将其转换为string，则必须使用该文件的编码。

.NET包括编码和解码类：ASCII，UTF7，UTF8，UTF32等等。

大多数这些编码包含某些字节顺序标记，可以用来区分使用哪种编码types。

.NET类System.IO.StreamReader能够通过读取这些字节顺序标记来确定stream中使用的编码;

这里是一个例子：

  /// <summary> /// return the detected encoding and the contents of the file. /// </summary> /// <param name="fileName"></param> /// <param name="contents"></param> /// <returns></returns> public static Encoding DetectEncoding(String fileName, out String contents) { // open the file with the stream-reader: using (StreamReader reader = new StreamReader(fileName, true)) { // read the contents of the file into a string contents = reader.ReadToEnd(); // return the encoding. return reader.CurrentEncoding; } } 

Answer 5

另一种select，很晚很晚，抱歉：

http://www.architectshack.com/TextFileEncodingDetector.ashx

这个小的C＃-only类使用BOMS（如果存在的话），否则会尝试自动检测可能的Unicode编码，如果Unicode编码都不可能或不可能，则会退步。

这听起来像上面引用的UTF8Checker做了类似的事情，但我认为这是范围稍宽 – 而不仅仅是UTF8，它还检查其他可能的Unicode编码（UTF-16 LE或BE），可能会丢失一个BOM。

希望这可以帮助别人！

Answer 6

我的解决scheme是使用内置的东西，一些fallback。

我从一个答案select策略从另一个类似的问题上，但我现在找不到它。

它首先使用StreamReader中的内置逻辑检查物料清单，如果有BOM，编码将是Encoding.Default以外的东西，我们应该相信这个结果。

如果不是，则检查字节序列是否是有效的UTF-8序列。 如果是，则会猜测UTF-8为编码，如果不是，则再次默认的ASCII编码是结果。

 static Encoding getEncoding(string path) { var stream = new FileStream(path, FileMode.Open); var reader = new StreamReader(stream, Encoding.Default, true); reader.Read(); if (reader.CurrentEncoding != Encoding.Default) { reader.Close(); return reader.CurrentEncoding; } stream.Position = 0; reader = new StreamReader(stream, new UTF8Encoding(false, true)); try { reader.ReadToEnd(); reader.Close(); return Encoding.UTF8; } catch (Exception) { reader.Close(); return Encoding.Default; } } 

Answer 7

SimpleHelpers.FileEncoding Nuget包将Mozilla Universal Charset Detector的C＃端口封装成简单的API：

 var encoding = FileEncoding.DetectFileEncoding(txtFile); 

Answer 8

我写了这段代码来区分UTF-8和Windows-1252。 它不应该用于巨大的文本文件，因为它将整个东西加载到内存中并完全扫描它。 我用它为.srt字幕文件，只是为了能够将它们保存在他们加载的编码。

如果文件被检测为不是有效的UTF-8，那么赋给函数ref的编码应该是8位的后备编码; 一般来说，在Windows系统上，这将是Windows-1252。 这虽然不会像检查实际有效的ASCII范围那样花哨，甚至在字节顺序标记上也不会检测到UTF-16。

按位检测背后的理论可以在这里find： https : //ianthehenry.com/2015/1/17/decoding-utf-8/

基本上，第一个字节的位范围决定了它作为UTF-8实体的一部分之后的位数。 它们之后的这些字节总是在相同的位范围内。

 /// <summary> /// Detects whether the encoding of the data is valid UTF-8 or ascii. If detection fails, the text is decoded using the given fallback encoding. /// Bit-wise mechanism for detecting valid UTF-8 based on https://ianthehenry.com/2015/1/17/decoding-utf-8/ /// Note that pure ascii detection should not be trusted: it might mean the file is meant to be UTF-8 or Windows-1252 but simply contains no special characters. /// </summary> /// <param name="docBytes">The bytes of the text document.</param> /// <param name="encoding">The default encoding to use as fallback if the text is detected not to be pure ascii or UTF-8 compliant. This ref parameter is changed to the detected encoding, or Windows-1252 if the given encoding parameter is null and the text is not valid UTF-8.</param> /// <returns>The contents of the read file</returns> public static String ReadFileAndGetEncoding(Byte[] docBytes, ref Encoding encoding) { if (encoding == null) encoding = Encoding.GetEncoding(1252); // BOM detection is not added in this example. Add it yourself if you feel like it. Should set the "encoding" param and return the decoded string. //String file = DetectByBOM(docBytes, ref encoding); //if (file != null) // return file; Boolean isPureAscii = true; Boolean isUtf8Valid = true; for (Int32 i = 0; i < docBytes.Length; i++) { Int32 skip = TestUtf8(docBytes, i); if (skip != 0) { if (isPureAscii) isPureAscii = false; if (skip < 0) isUtf8Valid = false; else i += skip; } // if already detected that it's not valid utf8, there's no sense in going on. if (!isUtf8Valid) break; } if (isPureAscii) encoding = new ASCIIEncoding(); // pure 7-bit ascii. else if (isUtf8Valid) encoding = new UTF8Encoding(false); // else, retain given fallback encoding. return encoding.GetString(docBytes); } /// <summary> /// Tests if the bytes following the given offset are UTF-8 valid, and returns /// the extra amount of bytes to skip ahead to do the next read if it is /// (meaning, detecting a single-byte ascii character would return 0). /// If the text is not UTF-8 valid it returns -1. /// </summary> /// <param name="binFile">Byte array to test</param> /// <param name="offset">Offset in the byte array to test.</param> /// <returns>The amount of extra bytes to skip ahead for the next read, or -1 if the byte sequence wasn't valid UTF-8</returns> public static Int32 TestUtf8(Byte[] binFile, Int32 offset) { Byte current = binFile[offset]; if ((current & 0x80) == 0) return 0; // valid 7-bit ascii. Added length is 0 bytes. else { Int32 len = binFile.Length; Int32 fullmask = 0xC0; Int32 testmask = 0; for (Int32 addedlength = 1; addedlength < 6; addedlength++) { // This code adds shifted bits to get the desired full mask. // If the full mask is [111]0 0000, then test mask will be [110]0 0000. Since this is // effectively always the previous step in the iteration I just store it each time. testmask = fullmask; fullmask += (0x40 >> addedlength); // Test bit mask for this level if ((current & fullmask) == testmask) { // End of file. Might be cut off, but either way, deemed invalid. if (offset + addedlength >= len) return -1; else { // Lookahead. Pattern of any following bytes is always 10xxxxxx for (Int32 i = 1; i <= addedlength; i++) { // If it does not match the pattern for an added byte, it is deemed invalid. if ((binFile[offset + i] & 0xC0) != 0x80) return -1; } return addedlength; } } } // Value is greater than the start of a 6-byte utf8 sequence. Deemed invalid. return -1; } }