diff --git a/ParserTools/FileReader.cs b/ParserTools/FileReader.cs
new file mode 100644
index 000000000..a1a7aacce
--- /dev/null
+++ b/ParserTools/FileReader.cs
@@ -0,0 +1,154 @@
+//
+//
+//
+//
+// $Revision: 2682 $
+//
+
+using System;
+using System.IO;
+using System.Text;
+
+namespace PascalABCCompiler
+{
+ ///
+ /// Class that can open text files with auto-detection of the encoding.
+ ///
+ public static class FileReader
+ {
+ public static bool IsUnicode(Encoding encoding)
+ {
+ int codepage = encoding.CodePage;
+ // return true if codepage is any UTF codepage
+ return codepage == 65001 || codepage == 65000 || codepage == 1200 || codepage == 1201;
+ }
+
+ public static string ReadFileContent(Stream fs, ref Encoding encoding)
+ {
+ using (StreamReader reader = OpenStream(fs, encoding)) {
+ reader.Peek();
+ encoding = reader.CurrentEncoding;
+ return reader.ReadToEnd();
+ }
+ }
+
+ public static string ReadFileContent(string fileName, Encoding encoding)
+ {
+ if (encoding == null)
+ encoding = System.Text.Encoding.GetEncoding(1251);
+ using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
+ return ReadFileContent(fs, ref encoding);
+ }
+ }
+
+ public static StreamReader OpenStream(Stream fs, Encoding defaultEncoding)
+ {
+ if (fs == null)
+ throw new ArgumentNullException("fs");
+
+ if (fs.Length >= 2) {
+ // the autodetection of StreamReader is not capable of detecting the difference
+ // between ISO-8859-1 and UTF-8 without BOM.
+ int firstByte = fs.ReadByte();
+ int secondByte = fs.ReadByte();
+ switch ((firstByte << 8) | secondByte) {
+ case 0x0000: // either UTF-32 Big Endian or a binary file; use StreamReader
+ case 0xfffe: // Unicode BOM (UTF-16 LE or UTF-32 LE)
+ case 0xfeff: // UTF-16 BE BOM
+ case 0xefbb: // start of UTF-8 BOM
+ // StreamReader autodetection works
+ fs.Position = 0;
+ return new StreamReader(fs);
+ default:
+ return AutoDetect(fs, (byte)firstByte, (byte)secondByte, defaultEncoding);
+ }
+ } else {
+ if (defaultEncoding != null) {
+ return new StreamReader(fs, defaultEncoding);
+ } else {
+ return new StreamReader(fs);
+ }
+ }
+ }
+
+ static StreamReader AutoDetect(Stream fs, byte firstByte, byte secondByte, Encoding defaultEncoding)
+ {
+ int max = (int)Math.Min(fs.Length, 500000); // look at max. 500 KB
+ const int ASCII = 0;
+ const int Error = 1;
+ const int UTF8 = 2;
+ const int UTF8Sequence = 3;
+ int state = ASCII;
+ int sequenceLength = 0;
+ byte b;
+ for (int i = 0; i < max; i++) {
+ if (i == 0) {
+ b = firstByte;
+ } else if (i == 1) {
+ b = secondByte;
+ } else {
+ b = (byte)fs.ReadByte();
+ }
+ if (b < 0x80) {
+ // normal ASCII character
+ if (state == UTF8Sequence) {
+ state = Error;
+ break;
+ }
+ } else if (b < 0xc0) {
+ // 10xxxxxx : continues UTF8 byte sequence
+ if (state == UTF8Sequence) {
+ --sequenceLength;
+ if (sequenceLength < 0) {
+ state = Error;
+ break;
+ } else if (sequenceLength == 0) {
+ state = UTF8;
+ }
+ } else {
+ state = Error;
+ break;
+ }
+ } else if (b >= 0xc2 && b < 0xf5) {
+ // beginning of byte sequence
+ if (state == UTF8 || state == ASCII) {
+ state = UTF8Sequence;
+ if (b < 0xe0) {
+ sequenceLength = 1; // one more byte following
+ } else if (b < 0xf0) {
+ sequenceLength = 2; // two more bytes following
+ } else {
+ sequenceLength = 3; // three more bytes following
+ }
+ } else {
+ state = Error;
+ break;
+ }
+ } else {
+ // 0xc0, 0xc1, 0xf5 to 0xff are invalid in UTF-8 (see RFC 3629)
+ state = Error;
+ break;
+ }
+ }
+ fs.Position = 0;
+ switch (state) {
+ case ASCII:
+ case Error:
+ // when the file seems to be ASCII or non-UTF8,
+ // we read it using the user-specified encoding so it is saved again
+ // using that encoding.
+ if (IsUnicode(defaultEncoding)) {
+ // the file is not Unicode, so don't read it using Unicode even if the
+ // user has choosen Unicode as the default encoding.
+
+ // If we don't do this, SD will end up always adding a Byte Order Mark
+ // to ASCII files.
+ defaultEncoding = Encoding.Default; // use system encoding instead
+ }
+ return new StreamReader(fs, defaultEncoding);
+ default:
+ return new StreamReader(fs);
+ }
+ }
+ }
+}
diff --git a/ParserTools/ParserTools.csproj b/ParserTools/ParserTools.csproj
index e130b2ab5..91744a9ce 100644
--- a/ParserTools/ParserTools.csproj
+++ b/ParserTools/ParserTools.csproj
@@ -63,6 +63,7 @@
+
diff --git a/ParserTools/Tools.cs b/ParserTools/Tools.cs
index d5593a8ee..8a6939cd4 100644
--- a/ParserTools/Tools.cs
+++ b/ParserTools/Tools.cs
@@ -21,10 +21,11 @@ namespace PascalABCCompiler
{
case SourceFileOperation.GetText:
if (!File.Exists(FileName)) return null;
- TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+ /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
//TextReader tr = new StreamReader(FileName, System.Text.Encoding.);
string Text = tr.ReadToEnd();
- tr.Close();
+ tr.Close();*/
+ string Text = FileReader.ReadFileContent(FileName, null);
return Text;
case SourceFileOperation.Exists:
return File.Exists(FileName);
diff --git a/TestSuite/CompilationSamples/PABCSystem.pas b/TestSuite/CompilationSamples/PABCSystem.pas
index c34c9779e..f142210b0 100644
--- a/TestSuite/CompilationSamples/PABCSystem.pas
+++ b/TestSuite/CompilationSamples/PABCSystem.pas
@@ -2971,16 +2971,6 @@ begin
left := sb.ToString;
end;
-/// Возвращает инверсию строки
-function string.Inverse(): string;
-begin
- var sb := new System.Text.StringBuilder(Self.Length);
- for var i:= Self.Length downto 1 do
- sb.Append(Self[i]);
- Result := sb.ToString;
-end;
-
-
//------------------------------------------------------------------------------
// Extension methods for BigInteger
@@ -7270,6 +7260,15 @@ begin
Result := Self.ToWords().Select(s -> StrToFloat(s)).ToArray();
end;
+/// Возвращает инверсию строки
+function string.Inverse(): string;
+begin
+ var sb := new System.Text.StringBuilder(Self.Length);
+ for var i:= Self.Length downto 1 do
+ sb.Append(Self[i]);
+ Result := sb.ToString;
+end;
+
//------------------------------------------------------------------------------
function GetEXEFileName: string;
begin
diff --git a/TestSuite/ansitest.pas b/TestSuite/ansitest.pas
new file mode 100644
index 000000000..41c4553f1
--- /dev/null
+++ b/TestSuite/ansitest.pas
@@ -0,0 +1,12 @@
+begin
+ var arr: array[''..''] of integer;
+ arr[''] := 2;
+ assert(arr[''] = 2);
+ var c := '';
+ var i := 0;
+ case c of
+ '': i := 3;
+ '': i := 5;
+ end;
+ assert(i = 5);
+end.
\ No newline at end of file
diff --git a/TestSuite/unicodetest.pas b/TestSuite/unicodetest.pas
new file mode 100644
index 000000000..119a1cd98
--- /dev/null
+++ b/TestSuite/unicodetest.pas
@@ -0,0 +1,13 @@
+//üя
+begin
+ var arr: array['а'..'я'] of integer;
+ arr['д'] := 2;
+ assert(arr['д'] = 2);
+ var c := 'д';
+ var i := 0;
+ case c of
+ 'л': i := 3;
+ 'д': i := 5;
+ end;
+ assert(i = 5);
+end.
\ No newline at end of file
diff --git a/VisualPascalABCNET/DS/VisualEnvironmentCompiler.cs b/VisualPascalABCNET/DS/VisualEnvironmentCompiler.cs
index 56114cd56..05978236e 100644
--- a/VisualPascalABCNET/DS/VisualEnvironmentCompiler.cs
+++ b/VisualPascalABCNET/DS/VisualEnvironmentCompiler.cs
@@ -249,9 +249,10 @@ namespace VisualPascalABC
return ed.Document.TextContent;
if (!File.Exists(FileName))
return null;
- TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+ /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
string Text = tr.ReadToEnd();
- tr.Close();
+ tr.Close();*/
+ string Text = PascalABCCompiler.FileReader.ReadFileContent(FileName, null);
return Text;
case PascalABCCompiler.SourceFileOperation.Exists:
if (tp != null)
diff --git a/pabcnetc/CommandConsoleCompiler.cs b/pabcnetc/CommandConsoleCompiler.cs
index 43ab72190..16cc1e97e 100644
--- a/pabcnetc/CommandConsoleCompiler.cs
+++ b/pabcnetc/CommandConsoleCompiler.cs
@@ -67,9 +67,10 @@ namespace PascalABCCompiler
return text;
if (!File.Exists(FileName))
return null;
- TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+ /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
text = tr.ReadToEnd();
- tr.Close();
+ tr.Close();*/
+ text = PascalABCCompiler.FileReader.ReadFileContent(FileName, null);
return text;
case SourceFileOperation.Exists:
if (SourceFiles.ContainsKey(fn))
diff --git a/pabcnetc/ConsoleCompiler.cs b/pabcnetc/ConsoleCompiler.cs
index 71275eeb5..6f8ed5bdf 100644
--- a/pabcnetc/ConsoleCompiler.cs
+++ b/pabcnetc/ConsoleCompiler.cs
@@ -360,7 +360,7 @@ namespace PascalABCCompiler
DateTime ldt = DateTime.Now;
Compiler = new PascalABCCompiler.Compiler(null,ChangeCompilerState);
//GC.Collect();
- WriteColorText(Compiler.Banner + "\nCopyright (c) 2005,2014 PascalABCCompiler Team\n", ConsoleColor.Black, ConsoleColor.Green);
+ WriteColorText(Compiler.Banner + "\nCopyright (c) 2005,2015 PascalABCCompiler Team\n", ConsoleColor.Black, ConsoleColor.Green);
Console.WriteLine("OK {0}ms", (DateTime.Now - ldt).TotalMilliseconds);
if (Compiler.SupportedSourceFiles.Length == 0)
WriteColorText(StringResourcesGet("ERROR_PARSERS_NOT_FOUND")+Environment.NewLine,ConsoleColor.Black,ConsoleColor.Red);
diff --git a/pabcnetc_clear/CommandConsoleCompiler.cs b/pabcnetc_clear/CommandConsoleCompiler.cs
index 7c6a1344d..18893742a 100644
--- a/pabcnetc_clear/CommandConsoleCompiler.cs
+++ b/pabcnetc_clear/CommandConsoleCompiler.cs
@@ -66,9 +66,10 @@ namespace PascalABCCompiler
return text;
if (!File.Exists(FileName))
return null;
- TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
+ /*TextReader tr = new StreamReader(FileName, System.Text.Encoding.GetEncoding(1251));
text = tr.ReadToEnd();
- tr.Close();
+ tr.Close();*/
+ text = PascalABCCompiler.FileReader.ReadFileContent(FileName, null);
return text;
case SourceFileOperation.Exists:
if (SourceFiles.ContainsKey(fn))