bug fix issue #30

ParserTools/FileReader.cs

@@ -0,0 +1,154 @@
+// <file>
+//     <copyright see="prj:///doc/copyright.txt"/>
+//     <license see="prj:///doc/license.txt"/>
+//     <owner name="Daniel Grunwald" email="daniel@danielgrunwald.de"/>
+//     <version>$Revision: 2682 $</version>
+// </file>
+
+using System;
+using System.IO;
+using System.Text;
+
+namespace PascalABCCompiler
+{
+	/// <summary>
+	/// Class that can open text files with auto-detection of the encoding.
+	/// </summary>
+	public static class FileReader
+	{
+		public static bool IsUnicode(Encoding encoding)
+		{
+			int codepage = encoding.CodePage;
+			// return true if codepage is any UTF codepage
+			return codepage == 65001 || codepage == 65000 || codepage == 1200 || codepage == 1201;
+		}
+		
+		public static string ReadFileContent(Stream fs, ref Encoding encoding)
+		{
+			using (StreamReader reader = OpenStream(fs, encoding)) {
+				reader.Peek();
+				encoding = reader.CurrentEncoding;
+				return reader.ReadToEnd();
+			}
+		}
+		
+		public static string ReadFileContent(string fileName, Encoding encoding)
+		{
+            if (encoding == null)
+                encoding = System.Text.Encoding.GetEncoding(1251);
+			using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
+				return ReadFileContent(fs, ref encoding);
+			}
+		}
+		
+		public static StreamReader OpenStream(Stream fs, Encoding defaultEncoding)
+		{
+			if (fs == null)
+				throw new ArgumentNullException("fs");
+			
+			if (fs.Length >= 2) {
+				// the autodetection of StreamReader is not capable of detecting the difference
+				// between ISO-8859-1 and UTF-8 without BOM.
+				int firstByte = fs.ReadByte();
+				int secondByte = fs.ReadByte();
+				switch ((firstByte << 8) | secondByte) {
+					case 0x0000: // either UTF-32 Big Endian or a binary file; use StreamReader
+					case 0xfffe: // Unicode BOM (UTF-16 LE or UTF-32 LE)
+					case 0xfeff: // UTF-16 BE BOM
+					case 0xefbb: // start of UTF-8 BOM
+						// StreamReader autodetection works
+						fs.Position = 0;
+						return new StreamReader(fs);
+					default:
+						return AutoDetect(fs, (byte)firstByte, (byte)secondByte, defaultEncoding);
+				}
+			} else {
+				if (defaultEncoding != null) {
+					return new StreamReader(fs, defaultEncoding);
+				} else {
+					return new StreamReader(fs);
+				}
+			}
+		}
+		
+		static StreamReader AutoDetect(Stream fs, byte firstByte, byte secondByte, Encoding defaultEncoding)
+		{
+			int max = (int)Math.Min(fs.Length, 500000); // look at max. 500 KB
+			const int ASCII = 0;
+			const int Error = 1;
+			const int UTF8  = 2;
+			const int UTF8Sequence = 3;
+			int state = ASCII;
+			int sequenceLength = 0;
+			byte b;
+			for (int i = 0; i < max; i++) {
+				if (i == 0) {
+					b = firstByte;
+				} else if (i == 1) {
+					b = secondByte;
+				} else {
+					b = (byte)fs.ReadByte();
+				}
+				if (b < 0x80) {
+					// normal ASCII character
+					if (state == UTF8Sequence) {
+						state = Error;
+						break;
+					}
+				} else if (b < 0xc0) {
+					// 10xxxxxx : continues UTF8 byte sequence
+					if (state == UTF8Sequence) {
+						--sequenceLength;
+						if (sequenceLength < 0) {
+							state = Error;
+							break;
+						} else if (sequenceLength == 0) {
+							state = UTF8;
+						}
+					} else {
+						state = Error;
+						break;
+					}
+				} else if (b >= 0xc2 && b < 0xf5) {
+					// beginning of byte sequence
+					if (state == UTF8 || state == ASCII) {
+						state = UTF8Sequence;
+						if (b < 0xe0) {
+							sequenceLength = 1; // one more byte following
+						} else if (b < 0xf0) {
+							sequenceLength = 2; // two more bytes following
+						} else {
+							sequenceLength = 3; // three more bytes following
+						}
+					} else {
+						state = Error;
+						break;
+					}
+				} else {
+					// 0xc0, 0xc1, 0xf5 to 0xff are invalid in UTF-8 (see RFC 3629)
+					state = Error;
+					break;
+				}
+			}
+			fs.Position = 0;
+			switch (state) {
+				case ASCII:
+				case Error:
+					// when the file seems to be ASCII or non-UTF8,
+					// we read it using the user-specified encoding so it is saved again
+					// using that encoding.
+					if (IsUnicode(defaultEncoding)) {
+						// the file is not Unicode, so don't read it using Unicode even if the
+						// user has choosen Unicode as the default encoding.
+						
+						// If we don't do this, SD will end up always adding a Byte Order Mark
+						// to ASCII files.
+						defaultEncoding = Encoding.Default; // use system encoding instead
+					}
+					return new StreamReader(fs, defaultEncoding);
+				default:
+					return new StreamReader(fs);
+			}
+		}
+	}
+}

ParserTools/ParserTools.csproj

@@ -63,6 +63,7 @@
     </Compile>
     <Compile Include="Config\AssemblyInfo.cs" />
     <Compile Include="DocumentationConstructor.cs" />
+    <Compile Include="FileReader.cs" />
     <Compile Include="ParsersController.cs" />
     <Compile Include="ParserTools\BaseParser.cs" />
     <Compile Include="ParserTools\IParser.cs" />

ParserTools/Tools.cs

@@ -21,10 +21,11 @@ namespace PascalABCCompiler
             {
                 case SourceFileOperation.GetText:
                     if (!File.Exists(FileName)) return null;
-                    TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+                    /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
                     //TextReader tr = new StreamReader(FileName, System.Text.Encoding.);
                     string Text = tr.ReadToEnd();
-                    tr.Close();
+                    tr.Close();*/
+                    string Text = FileReader.ReadFileContent(FileName, null);
                     return Text;
                 case SourceFileOperation.Exists:
                     return File.Exists(FileName);

TestSuite/CompilationSamples/PABCSystem.pas

@@ -2971,16 +2971,6 @@ begin
   left := sb.ToString;
 end;
 
-/// Возвращает инверсию строки
-function string.Inverse(): string;
-begin
-  var sb := new System.Text.StringBuilder(Self.Length);
-  for var i:= Self.Length downto 1 do
-    sb.Append(Self[i]);
-  Result := sb.ToString;
-end;
-
-
 
 //------------------------------------------------------------------------------
 // Extension methods for BigInteger
@@ -7270,6 +7260,15 @@ begin
   Result := Self.ToWords().Select(s -> StrToFloat(s)).ToArray();
 end;
 
+/// Возвращает инверсию строки
+function string.Inverse(): string;
+begin
+  var sb := new System.Text.StringBuilder(Self.Length);
+  for var i:= Self.Length downto 1 do
+    sb.Append(Self[i]);
+  Result := sb.ToString;
+end;
+
 //------------------------------------------------------------------------------
 function GetEXEFileName: string;
 begin

TestSuite/ansitest.pas

@@ -0,0 +1,12 @@
+begin
+  var arr: array['à'..'ÿ'] of integer;
+  arr['ä'] := 2;
+  assert(arr['ä'] = 2);
+  var c := 'ä';
+  var i := 0;
+  case c of
+    'ë': i := 3;
+    'ä': i := 5;
+  end;
+  assert(i = 5);
+end.

TestSuite/unicodetest.pas

@@ -0,0 +1,13 @@
+//üя
+begin
+  var arr: array['а'..'я'] of integer;
+  arr['д'] := 2;
+  assert(arr['д'] = 2);
+  var c := 'д';
+  var i := 0;
+  case c of
+    'л': i := 3;
+    'д': i := 5;
+  end;
+  assert(i = 5);
+end.

VisualPascalABCNET/DS/VisualEnvironmentCompiler.cs

@@ -249,9 +249,10 @@ namespace VisualPascalABC
                         return ed.Document.TextContent;
                     if (!File.Exists(FileName))
                         return null;
-                    TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+                    /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
                     string Text = tr.ReadToEnd();
-                    tr.Close();
+                    tr.Close();*/
+                    string Text = PascalABCCompiler.FileReader.ReadFileContent(FileName, null);
                     return Text;
                 case PascalABCCompiler.SourceFileOperation.Exists:
                     if (tp != null)

pabcnetc/CommandConsoleCompiler.cs

@@ -67,9 +67,10 @@ namespace PascalABCCompiler
                         return text;
                     if (!File.Exists(FileName))
                         return null;
-                    TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+                    /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
                     text = tr.ReadToEnd();
-                    tr.Close();
+                    tr.Close();*/
+                    text = PascalABCCompiler.FileReader.ReadFileContent(FileName, null);
                     return text;
                 case SourceFileOperation.Exists:
                     if (SourceFiles.ContainsKey(fn))

pabcnetc/ConsoleCompiler.cs

@@ -360,7 +360,7 @@ namespace PascalABCCompiler
             DateTime ldt = DateTime.Now;
             Compiler = new PascalABCCompiler.Compiler(null,ChangeCompilerState);
             //GC.Collect();
-            WriteColorText(Compiler.Banner + "\nCopyright (c) 2005,2014 PascalABCCompiler Team\n", ConsoleColor.Black, ConsoleColor.Green);
+            WriteColorText(Compiler.Banner + "\nCopyright (c) 2005,2015 PascalABCCompiler Team\n", ConsoleColor.Black, ConsoleColor.Green);
             Console.WriteLine("OK {0}ms", (DateTime.Now - ldt).TotalMilliseconds);
             if (Compiler.SupportedSourceFiles.Length == 0)
                 WriteColorText(StringResourcesGet("ERROR_PARSERS_NOT_FOUND")+Environment.NewLine,ConsoleColor.Black,ConsoleColor.Red);

pabcnetc_clear/CommandConsoleCompiler.cs

@@ -66,9 +66,10 @@ namespace PascalABCCompiler
                         return text;
                     if (!File.Exists(FileName))
                         return null;
-                    TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+                    /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
                     text = tr.ReadToEnd();
-                    tr.Close();
+                    tr.Close();*/
+                    text = PascalABCCompiler.FileReader.ReadFileContent(FileName, null);
                     return text;
                 case SourceFileOperation.Exists:
                     if (SourceFiles.ContainsKey(fn))