d5e9994ff2
* Return Parsers folder with standard parsers and move SyntaxTreeConverters back to Core * Rename PascalLanguage project to PascalABCLanguageInfo * Fix pascal dll loading
1679 lines
57 KiB
C#
1679 lines
57 KiB
C#
//
|
|
// This CSharp output file generated by Gardens Point LEX
|
|
// Version: 1.1.3.301
|
|
// Machine: MAINCOMPUTER
|
|
// DateTime: 29.07.2010 22:24:20
|
|
// UserName: JCommander
|
|
// GPLEX input file <oberon00.lex>
|
|
// GPLEX frame file <embedded resource>
|
|
//
|
|
// Option settings: unicode, parser, minimize
|
|
// Option settings: classes, compressMap, compressNext, persistBuffer, embedbuffers
|
|
// Fallback code page: Target machine default
|
|
//
|
|
|
|
//
|
|
// Experimental embedded frame
|
|
// Version 1.1.3 of 18-April-2010
|
|
//
|
|
//
|
|
#define BACKUP
|
|
#define PERSIST
|
|
|
|
using System;
|
|
using System.IO;
|
|
using System.Text;
|
|
using System.Globalization;
|
|
using System.Collections.Generic;
|
|
using System.Runtime.Serialization;
|
|
using System.Diagnostics.CodeAnalysis;
|
|
|
|
using PascalABCCompiler.Oberon00Parser;
|
|
|
|
namespace GPPGParserScanner
|
|
{
|
|
/// <summary>
|
|
/// Summary Canonical example of GPLEX automaton
|
|
/// </summary>
|
|
|
|
#if STANDALONE
|
|
//
|
|
// These are the dummy declarations for stand-alone GPLEX applications
|
|
// normally these declarations would come from the parser.
|
|
// If you declare /noparser, or %option noparser then you get this.
|
|
//
|
|
|
|
public enum Tokens
|
|
{
|
|
EOF = 0, maxParseToken = int.MaxValue
|
|
// must have at least these two, values are almost arbitrary
|
|
}
|
|
|
|
public abstract class ScanBase
|
|
{
|
|
[SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")]
|
|
public abstract int yylex();
|
|
|
|
[SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yywrap")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yywrap")]
|
|
protected virtual bool yywrap() { return true; }
|
|
|
|
#if BABEL
|
|
protected abstract int CurrentSc { get; set; }
|
|
// EolState is the 32-bit of state data persisted at
|
|
// the end of each line for Visual Studio colorization.
|
|
// The default is to return CurrentSc. You must override
|
|
// this if you want more complicated behavior.
|
|
public virtual int EolState {
|
|
get { return CurrentSc; }
|
|
set { CurrentSc = value; }
|
|
}
|
|
}
|
|
|
|
public interface IColorScan
|
|
{
|
|
void SetSource(string source, int offset);
|
|
int GetNext(ref int state, out int start, out int end);
|
|
#endif // BABEL
|
|
}
|
|
|
|
#endif // STANDALONE
|
|
|
|
// If the compiler can't find the scanner base class maybe you
|
|
// need to run GPPG with the /gplex option, or GPLEX with /noparser
|
|
#if BABEL
|
|
public sealed partial class Scanner : ScanBase, IColorScan
|
|
{
|
|
private ScanBuff buffer;
|
|
int currentScOrd; // start condition ordinal
|
|
|
|
protected override int CurrentSc
|
|
{
|
|
// The current start state is a property
|
|
// to try to avoid the user error of setting
|
|
// scState but forgetting to update the FSA
|
|
// start state "currentStart"
|
|
//
|
|
get { return currentScOrd; } // i.e. return YY_START;
|
|
set { currentScOrd = value; // i.e. BEGIN(value);
|
|
currentStart = startState[value]; }
|
|
}
|
|
#else // BABEL
|
|
public sealed partial class Scanner : ScanBase
|
|
{
|
|
private ScanBuff buffer;
|
|
int currentScOrd; // start condition ordinal
|
|
#endif // BABEL
|
|
|
|
/// <summary>
|
|
/// The input buffer for this scanner.
|
|
/// </summary>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
public ScanBuff Buffer { get { return buffer; } }
|
|
|
|
private static int GetMaxParseToken() {
|
|
System.Reflection.FieldInfo f = typeof(Tokens).GetField("maxParseToken");
|
|
return (f == null ? int.MaxValue : (int)f.GetValue(null));
|
|
}
|
|
|
|
static int parserMax = GetMaxParseToken();
|
|
|
|
enum Result {accept, noMatch, contextFound};
|
|
|
|
const int maxAccept = 36;
|
|
const int initial = 37;
|
|
const int eofNum = 0;
|
|
const int goStart = -1;
|
|
const int INITIAL = 0;
|
|
const int COMMENT = 1;
|
|
const int COMMENT_S = 2;
|
|
|
|
#region user code
|
|
#endregion user code
|
|
|
|
int state;
|
|
int currentStart = startState[0];
|
|
int code; // last code read
|
|
int cCol; // column number of code
|
|
int lNum; // current line number
|
|
//
|
|
// The following instance variables are used, among other
|
|
// things, for constructing the yylloc location objects.
|
|
//
|
|
int tokPos; // buffer position at start of token
|
|
int tokCol; // zero-based column number at start of token
|
|
int tokLin; // line number at start of token
|
|
int tokEPos; // buffer position at end of token
|
|
int tokECol; // column number at end of token
|
|
int tokELin; // line number at end of token
|
|
string tokTxt; // lazily constructed text of token
|
|
#if STACK
|
|
private Stack<int> scStack = new Stack<int>();
|
|
#endif // STACK
|
|
|
|
#region ScannerTables
|
|
struct Table {
|
|
public int min; public int rng; public int dflt;
|
|
public sbyte[] nxt;
|
|
public Table(int m, int x, int d, sbyte[] n) {
|
|
min = m; rng = x; dflt = d; nxt = n;
|
|
}
|
|
};
|
|
|
|
static int[] startState = new int[] {37, 47, 50, 0};
|
|
|
|
#region CompressedCharacterMap
|
|
//
|
|
// There are 30 equivalence classes
|
|
// There are 2 character sequence regions
|
|
// There are 1 tables, 127 entries
|
|
// There are 1 runs, 0 singletons
|
|
// Decision tree depth is 1
|
|
//
|
|
static sbyte[] mapC0 = new sbyte[127] {
|
|
/* '\0' */ 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0,
|
|
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '\x20' */ 0, 17, 28, 10, 0, 0, 15, 27, 11, 12, 6, 5, 13, 4, 16, 7,
|
|
/* '0' */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 1, 3, 8, 2, 9, 0,
|
|
/* '@' */ 0, 24, 24, 24, 26, 26, 24, 22, 25, 22, 22, 22, 22, 22, 22, 22,
|
|
/* 'P' */ 22, 22, 22, 22, 22, 22, 22, 22, 29, 22, 22, 0, 0, 0, 0, 22,
|
|
/* '`' */ 0, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
|
|
/* 'p' */ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 18, 0, 19, 14 };
|
|
|
|
static sbyte MapC(int code)
|
|
{ // '\0' <= code <= '\U0010FFFF'
|
|
if (code < 127) // '\0' <= code <= '~'
|
|
return mapC0[code - 0];
|
|
else // '\x7F' <= code <= '\U0010FFFF'
|
|
return (sbyte)0;
|
|
}
|
|
#endregion
|
|
|
|
static Table[] NxS = new Table[51] {
|
|
/* NxS[ 0] */ new Table(0, 0, 0, null),
|
|
/* NxS[ 1] */ new Table(2, 1, -1, new sbyte[] {33}),
|
|
/* NxS[ 2] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 3] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 4] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 5] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 6] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 7] */ new Table(7, 1, -1, new sbyte[] {32}),
|
|
/* NxS[ 8] */ new Table(2, 1, -1, new sbyte[] {31}),
|
|
/* NxS[ 9] */ new Table(2, 1, -1, new sbyte[] {30}),
|
|
/* NxS[ 10] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 11] */ new Table(6, 1, -1, new sbyte[] {29}),
|
|
/* NxS[ 12] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 13] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 14] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 15] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 16] */ new Table(16, 1, -1, new sbyte[] {28}),
|
|
/* NxS[ 17] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 18] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 19] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 20] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 21] */ new Table(22, 8, -1, new sbyte[] {21, 21, 21, 21, 21, -1,
|
|
-1, 21}),
|
|
/* NxS[ 22] */ new Table(16, 14, -1, new sbyte[] {25, -1, -1, -1, -1, -1,
|
|
-1, 22, 44, 26, 44, -1, -1, 24}),
|
|
/* NxS[ 23] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 24] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 25] */ new Table(23, 4, -1, new sbyte[] {25, -1, -1, 45}),
|
|
/* NxS[ 26] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 27] */ new Table(23, 1, -1, new sbyte[] {27}),
|
|
/* NxS[ 28] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 29] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 30] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 31] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 32] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 33] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 34] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 35] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 36] */ new Table(0, 0, -1, null),
|
|
/* NxS[ 37] */ new Table(27, 27, 21, new sbyte[] {38, 39, 21, -1, 1, 2,
|
|
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
|
|
19, 20, -1, 21, 22}),
|
|
/* NxS[ 38] */ new Table(21, 7, 42, new sbyte[] {-1, 42, 42, 42, 42, 42,
|
|
23}),
|
|
/* NxS[ 39] */ new Table(21, 8, 40, new sbyte[] {-1, 40, 40, 40, 40, 40,
|
|
40, 23}),
|
|
/* NxS[ 40] */ new Table(21, 8, 41, new sbyte[] {-1, 41, 41, 41, 41, 41,
|
|
41, 24}),
|
|
/* NxS[ 41] */ new Table(21, 8, 41, new sbyte[] {-1, 41, 41, 41, 41, 41,
|
|
41, 23}),
|
|
/* NxS[ 42] */ new Table(21, 7, 43, new sbyte[] {-1, 43, 43, 43, 43, 43,
|
|
24}),
|
|
/* NxS[ 43] */ new Table(21, 7, 43, new sbyte[] {-1, 43, 43, 43, 43, 43,
|
|
23}),
|
|
/* NxS[ 44] */ new Table(23, 7, -1, new sbyte[] {44, 44, 26, 44, -1, -1,
|
|
24}),
|
|
/* NxS[ 45] */ new Table(23, 13, -1, new sbyte[] {27, -1, -1, -1, -1, -1,
|
|
-1, -1, -1, -1, -1, 46, 46}),
|
|
/* NxS[ 46] */ new Table(23, 1, -1, new sbyte[] {27}),
|
|
/* NxS[ 47] */ new Table(6, 6, -1, new sbyte[] {48, -1, -1, -1, -1, 49}),
|
|
/* NxS[ 48] */ new Table(12, 1, -1, new sbyte[] {35}),
|
|
/* NxS[ 49] */ new Table(6, 1, -1, new sbyte[] {34}),
|
|
/* NxS[ 50] */ new Table(21, 1, -1, new sbyte[] {36}),
|
|
};
|
|
|
|
int NextState() {
|
|
if (code == ScanBuff.EndOfFile)
|
|
return eofNum;
|
|
else
|
|
unchecked {
|
|
int rslt;
|
|
int idx = MapC(code) - NxS[state].min;
|
|
if (idx < 0) idx += 30;
|
|
if ((uint)idx >= (uint)NxS[state].rng) rslt = NxS[state].dflt;
|
|
else rslt = NxS[state].nxt[idx];
|
|
return rslt;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
|
|
#if BACKUP
|
|
// ==============================================================
|
|
// == Nested struct used for backup in automata that do backup ==
|
|
// ==============================================================
|
|
|
|
struct Context // class used for automaton backup.
|
|
{
|
|
public int bPos;
|
|
public int rPos; // scanner.readPos saved value
|
|
public int cCol;
|
|
public int lNum; // Need this in case of backup over EOL.
|
|
public int state;
|
|
public int cChr;
|
|
}
|
|
|
|
private Context ctx = new Context();
|
|
#endif // BACKUP
|
|
|
|
// ==============================================================
|
|
// ==== Nested struct to support input switching in scanners ====
|
|
// ==============================================================
|
|
|
|
struct BufferContext {
|
|
internal ScanBuff buffSv;
|
|
internal int chrSv;
|
|
internal int cColSv;
|
|
internal int lNumSv;
|
|
}
|
|
|
|
// ==============================================================
|
|
// ===== Private methods to save and restore buffer contexts ====
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This method creates a buffer context record from
|
|
/// the current buffer object, together with some
|
|
/// scanner state values.
|
|
/// </summary>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
BufferContext MkBuffCtx()
|
|
{
|
|
BufferContext rslt;
|
|
rslt.buffSv = this.buffer;
|
|
rslt.chrSv = this.code;
|
|
rslt.cColSv = this.cCol;
|
|
rslt.lNumSv = this.lNum;
|
|
return rslt;
|
|
}
|
|
|
|
/// <summary>
|
|
/// This method restores the buffer value and allied
|
|
/// scanner state from the given context record value.
|
|
/// </summary>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
void RestoreBuffCtx(BufferContext value)
|
|
{
|
|
this.buffer = value.buffSv;
|
|
this.code = value.chrSv;
|
|
this.cCol = value.cColSv;
|
|
this.lNum = value.lNumSv;
|
|
}
|
|
// =================== End Nested classes =======================
|
|
|
|
#if !NOFILES
|
|
public Scanner(Stream file) {
|
|
SetSource(file, 0); // unicode option
|
|
}
|
|
|
|
public Scanner(Stream file, string codepage) {
|
|
SetSource(file, CodePageHandling.GetCodePage(codepage));
|
|
}
|
|
|
|
#endif // !NOFILES
|
|
|
|
public Scanner() { }
|
|
|
|
private int readPos;
|
|
|
|
void GetCode()
|
|
{
|
|
if (code == '\n') // This needs to be fixed for other conventions
|
|
// i.e. [\r\n\205\u2028\u2029]
|
|
{
|
|
cCol = -1;
|
|
lNum++;
|
|
}
|
|
readPos = buffer.Pos;
|
|
|
|
// Now read new codepoint.
|
|
code = buffer.Read();
|
|
if (code > ScanBuff.EndOfFile)
|
|
{
|
|
#if (!BYTEMODE)
|
|
if (code >= 0xD800 && code <= 0xDBFF)
|
|
{
|
|
int next = buffer.Read();
|
|
if (next < 0xDC00 || next > 0xDFFF)
|
|
code = ScanBuff.UnicodeReplacementChar;
|
|
else
|
|
code = (0x10000 + (code & 0x3FF << 10) + (next & 0x3FF));
|
|
}
|
|
#endif
|
|
cCol++;
|
|
}
|
|
}
|
|
|
|
void MarkToken()
|
|
{
|
|
#if (!PERSIST)
|
|
buffer.Mark();
|
|
#endif
|
|
tokPos = readPos;
|
|
tokLin = lNum;
|
|
tokCol = cCol;
|
|
}
|
|
|
|
void MarkEnd()
|
|
{
|
|
tokTxt = null;
|
|
tokEPos = readPos;
|
|
tokELin = lNum;
|
|
tokECol = cCol;
|
|
}
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
int Peek()
|
|
{
|
|
int rslt, codeSv = code, cColSv = cCol, lNumSv = lNum, bPosSv = buffer.Pos;
|
|
GetCode(); rslt = code;
|
|
lNum = lNumSv; cCol = cColSv; code = codeSv; buffer.Pos = bPosSv;
|
|
return rslt;
|
|
}
|
|
|
|
// ==============================================================
|
|
// ===== Initialization of string-based input buffers ====
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a StringBuff buffer object for this scanner
|
|
/// </summary>
|
|
/// <param name="source">the input string</param>
|
|
/// <param name="offset">starting offset in the string</param>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
public void SetSource(string source, int offset)
|
|
{
|
|
this.buffer = ScanBuff.GetBuffer(source);
|
|
this.buffer.Pos = offset;
|
|
this.lNum = 0;
|
|
this.code = '\n'; // to initialize yyline, yycol and lineStart
|
|
GetCode();
|
|
}
|
|
|
|
#if !NOFILES
|
|
// ================ LineBuffer Initialization ===================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a LineBuff buffer object for this scanner
|
|
/// </summary>
|
|
/// <param name="source">the list of input strings</param>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
public void SetSource(IList<string> source)
|
|
{
|
|
this.buffer = ScanBuff.GetBuffer(source);
|
|
this.code = '\n'; // to initialize yyline, yycol and lineStart
|
|
this.lNum = 0;
|
|
GetCode();
|
|
}
|
|
|
|
// =============== StreamBuffer Initialization ==================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a StreamBuff buffer object for this scanner.
|
|
/// StreamBuff is buffer for 8-bit byte files.
|
|
/// </summary>
|
|
/// <param name="source">the input byte stream</param>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
public void SetSource(Stream source)
|
|
{
|
|
this.buffer = ScanBuff.GetBuffer(source);
|
|
this.lNum = 0;
|
|
this.code = '\n'; // to initialize yyline, yycol and lineStart
|
|
GetCode();
|
|
}
|
|
|
|
#if !BYTEMODE
|
|
// ================ TextBuffer Initialization ===================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a TextBuff buffer object for this scanner.
|
|
/// TextBuff is a buffer for encoded unicode files.
|
|
/// </summary>
|
|
/// <param name="source">the input text file</param>
|
|
/// <param name="fallbackCodePage">Code page to use if file has
|
|
/// no BOM. For 0, use machine default; for -1, 8-bit binary</param>
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
public void SetSource(Stream source, int fallbackCodePage)
|
|
{
|
|
this.buffer = ScanBuff.GetBuffer(source, fallbackCodePage);
|
|
this.lNum = 0;
|
|
this.code = '\n'; // to initialize yyline, yycol and lineStart
|
|
GetCode();
|
|
}
|
|
#endif // !BYTEMODE
|
|
#endif // !NOFILES
|
|
|
|
// ==============================================================
|
|
|
|
#if BABEL
|
|
//
|
|
// Get the next token for Visual Studio
|
|
//
|
|
// "state" is the inout mode variable that maintains scanner
|
|
// state between calls, using the EolState property. In principle,
|
|
// if the calls of EolState are costly set could be called once
|
|
// only per line, at the start; and get called only at the end
|
|
// of the line. This needs more infrastructure ...
|
|
//
|
|
public int GetNext(ref int state, out int start, out int end)
|
|
{
|
|
Tokens next;
|
|
int s, e;
|
|
s = state; // state at start
|
|
EolState = state;
|
|
next = (Tokens)Scan();
|
|
state = EolState;
|
|
e = state; // state at end;
|
|
start = tokPos;
|
|
end = tokEPos - 1; // end is the index of last char.
|
|
return (int)next;
|
|
}
|
|
#endif // BABEL
|
|
|
|
// ======== AbstractScanner<> Implementation =========
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")]
|
|
public override int yylex()
|
|
{
|
|
// parserMax is set by reflecting on the Tokens
|
|
// enumeration. If maxParseToken is defined
|
|
// that is used, otherwise int.MaxValue is used.
|
|
int next;
|
|
do { next = Scan(); } while (next >= parserMax);
|
|
return next;
|
|
}
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
int yypos { get { return tokPos; } }
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
int yyline { get { return tokLin; } }
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
int yycol { get { return tokCol; } }
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yytext")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yytext")]
|
|
public string yytext
|
|
{
|
|
get
|
|
{
|
|
if (tokTxt == null)
|
|
tokTxt = buffer.GetString(tokPos, tokEPos);
|
|
return tokTxt;
|
|
}
|
|
}
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
void yyless(int n)
|
|
{
|
|
buffer.Pos = tokPos;
|
|
// Must read at least one char, so set before start.
|
|
cCol = tokCol - 1;
|
|
GetCode();
|
|
// Now ensure that line counting is correct.
|
|
lNum = tokLin;
|
|
// And count the rest of the text.
|
|
for (int i = 0; i < n; i++) GetCode();
|
|
MarkEnd();
|
|
}
|
|
|
|
//
|
|
// It would be nice to count backward in the text
|
|
// but it does not seem possible to re-establish
|
|
// the correct column counts except by going forward.
|
|
//
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
void _yytrunc(int n) { yyless(yyleng - n); }
|
|
|
|
//
|
|
// This is painful, but we no longer count
|
|
// codepoints. For the overwhelming majority
|
|
// of cases the single line code is fast, for
|
|
// the others, well, at least it is all in the
|
|
// buffer so no files are touched. Note that we
|
|
// can't use (tokEPos - tokPos) because of the
|
|
// possibility of surrogate pairs in the token.
|
|
//
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyleng")]
|
|
[SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyleng")]
|
|
public int yyleng
|
|
{
|
|
get {
|
|
#if BYTEMODE
|
|
return tokEPos - tokPos;
|
|
#else
|
|
if (tokELin == tokLin)
|
|
return tokECol - tokCol;
|
|
else {
|
|
int ch;
|
|
int count = 0;
|
|
int save = buffer.Pos;
|
|
buffer.Pos = tokPos;
|
|
do {
|
|
ch = buffer.Read();
|
|
if (!char.IsHighSurrogate((char)ch)) count++;
|
|
} while (buffer.Pos < tokEPos && ch != ScanBuff.EndOfFile);
|
|
buffer.Pos = save;
|
|
return count;
|
|
}
|
|
#endif // BYTEMODE
|
|
}
|
|
}
|
|
|
|
// ============ methods available in actions ==============
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
internal int YY_START {
|
|
get { return currentScOrd; }
|
|
set { currentScOrd = value;
|
|
currentStart = startState[value];
|
|
}
|
|
}
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
internal void BEGIN(int next) {
|
|
currentScOrd = next;
|
|
currentStart = startState[next];
|
|
}
|
|
|
|
// ============== The main tokenizer code =================
|
|
|
|
int Scan()
|
|
{
|
|
try {
|
|
for (; ; )
|
|
{
|
|
int next; // next state to enter
|
|
#if BACKUP
|
|
Result rslt = Result.noMatch;
|
|
#endif // BACKUP
|
|
#if LEFTANCHORS
|
|
for (;;)
|
|
{
|
|
// Discard characters that do not start any pattern.
|
|
// Must check the left anchor condition after *every* GetCode!
|
|
state = ((cCol == 0) ? anchorState[currentScOrd] : currentStart);
|
|
if ((next = NextState()) != goStart)
|
|
break; // LOOP EXIT HERE...
|
|
GetCode();
|
|
}
|
|
|
|
#else // !LEFTANCHORS
|
|
state = currentStart;
|
|
while ((next = NextState()) == goStart)
|
|
// At this point, the current character has no
|
|
// transition from the current state. We discard
|
|
// the "no-match" char. In traditional LEX such
|
|
// characters are echoed to the console.
|
|
GetCode();
|
|
#endif // LEFTANCHORS
|
|
// At last, a valid transition ...
|
|
MarkToken();
|
|
state = next;
|
|
GetCode();
|
|
|
|
while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum
|
|
#if BACKUP
|
|
if (state <= maxAccept && next > maxAccept) // need to prepare backup data
|
|
{
|
|
// ctx is an object. The fields may be
|
|
// mutated by the call to Recurse2.
|
|
// On return the data in ctx is the
|
|
// *latest* accept state that was found.
|
|
|
|
rslt = Recurse2(ref ctx, next);
|
|
if (rslt == Result.noMatch)
|
|
RestoreStateAndPos(ref ctx);
|
|
break;
|
|
}
|
|
else
|
|
#endif // BACKUP
|
|
{
|
|
state = next;
|
|
GetCode();
|
|
}
|
|
if (state <= maxAccept)
|
|
{
|
|
MarkEnd();
|
|
#region ActionSwitch
|
|
#pragma warning disable 162
|
|
switch (state)
|
|
{
|
|
case eofNum:
|
|
switch (currentStart) {
|
|
case 47:
|
|
PT.AddError("Комментарий не закрыт", yylloc);
|
|
break;
|
|
}
|
|
if (yywrap())
|
|
return (int)Tokens.EOF;
|
|
break;
|
|
case 1:
|
|
return (int)Tokens.COLON;
|
|
break;
|
|
case 2:
|
|
return (int)Tokens.EQ;
|
|
break;
|
|
case 3:
|
|
return (int)Tokens.SEMICOLUMN;
|
|
break;
|
|
case 4:
|
|
return (int)Tokens.MINUS;
|
|
break;
|
|
case 5:
|
|
return (int)Tokens.PLUS;
|
|
break;
|
|
case 6:
|
|
return (int)Tokens.MULT;
|
|
break;
|
|
case 7:
|
|
return (int)Tokens.DIVIDE;
|
|
break;
|
|
case 8:
|
|
return (int)Tokens.LT;
|
|
break;
|
|
case 9:
|
|
return (int)Tokens.GT;
|
|
break;
|
|
case 10:
|
|
return (int)Tokens.NE;
|
|
break;
|
|
case 11:
|
|
return (int)Tokens.LPAREN;
|
|
break;
|
|
case 12:
|
|
return (int)Tokens.RPAREN;
|
|
break;
|
|
case 13:
|
|
return (int)Tokens.COLUMN;
|
|
break;
|
|
case 14:
|
|
return (int)Tokens.NOT;
|
|
break;
|
|
case 15:
|
|
return (int)Tokens.AND;
|
|
break;
|
|
case 16:
|
|
return (int)Tokens.COMMA;
|
|
break;
|
|
case 17:
|
|
return (int)Tokens.EXCLAMATION;
|
|
break;
|
|
case 18:
|
|
return (int)Tokens.LBRACE;
|
|
break;
|
|
case 19:
|
|
return (int)Tokens.RBRACE;
|
|
break;
|
|
case 20:
|
|
return (int)Tokens.INVISIBLE;
|
|
break;
|
|
case 21:
|
|
int res = Keywords.KeywordOrIDToken(yytext);
|
|
if (res == (int)Tokens.ID){
|
|
yylval.sVal = yytext;
|
|
PT.LastIdentificator = yytext;
|
|
}
|
|
return res;
|
|
break;
|
|
case 22:
|
|
int tryParseInt;
|
|
if (int.TryParse(yytext, out tryParseInt)){
|
|
yylval.iVal = tryParseInt;
|
|
return (int)Tokens.INTNUM;
|
|
}
|
|
System.Int64 tryParseLong;
|
|
if (System.Int64.TryParse(yytext, out tryParseLong)){
|
|
yylval.lVal = tryParseLong;
|
|
return (int)Tokens.LONGINTNUM;
|
|
}
|
|
PT.AddError("Слишком длинное целое", yylloc);
|
|
break;
|
|
case 23:
|
|
yylval.sVal = PT.GetStringContent(yytext);
|
|
return (int)Tokens.STRING_CONST;
|
|
break;
|
|
case 24:
|
|
try{
|
|
yylval.cVal = PT.GetStringContent(yytext)[0];
|
|
return (int)Tokens.CHAR_CONST;
|
|
}
|
|
catch (System.ArgumentException){
|
|
PT.AddError("Некорректный код символа", yylloc);
|
|
}
|
|
break;
|
|
case 25:
|
|
case 27:
|
|
double tryParseDouble;
|
|
var correctDouble = PT.GetCorrectDoubleStr(yytext);
|
|
if (double.TryParse(correctDouble, out tryParseDouble)){
|
|
yylval.rVal = tryParseDouble;
|
|
return (int)Tokens.REALNUM;
|
|
}
|
|
PT.AddError("Слишком длинное вещественное", yylloc);
|
|
break;
|
|
case 26:
|
|
var _yytext = yytext.Substring(0, yytext.Length - 1);
|
|
int tryParseHexInt;
|
|
if (int.TryParse(_yytext, System.Globalization.NumberStyles.AllowHexSpecifier, null, out tryParseHexInt)){
|
|
yylval.iVal = tryParseHexInt;
|
|
return (int)Tokens.INTNUM;
|
|
}
|
|
System.Int64 tryParseHexLong;
|
|
if (System.Int64.TryParse(_yytext, System.Globalization.NumberStyles.AllowHexSpecifier, null, out tryParseHexLong)){
|
|
yylval.lVal = tryParseHexLong;
|
|
return (int)Tokens.LONGINTNUM;
|
|
}
|
|
PT.AddError("Слишком длинное шестнадцатеричное целое", yylloc);
|
|
break;
|
|
case 28:
|
|
return (int)Tokens.DOUBLEPOINT;
|
|
break;
|
|
case 29:
|
|
BEGIN(COMMENT);
|
|
mlCommentCnt = 1;
|
|
break;
|
|
case 30:
|
|
return (int)Tokens.GE;
|
|
break;
|
|
case 31:
|
|
return (int)Tokens.LE;
|
|
break;
|
|
case 32:
|
|
BEGIN(COMMENT_S);
|
|
break;
|
|
case 33:
|
|
return (int)Tokens.ASSIGN;
|
|
break;
|
|
case 34:
|
|
++mlCommentCnt;
|
|
break;
|
|
case 35:
|
|
--mlCommentCnt;
|
|
if (mlCommentCnt == 0)
|
|
BEGIN(INITIAL);
|
|
break;
|
|
case 36:
|
|
BEGIN(INITIAL);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
#pragma warning restore 162
|
|
#endregion
|
|
}
|
|
}
|
|
} // end try
|
|
finally {
|
|
// User-specified epilog to scan()
|
|
yylloc = new QUT.Gppg.LexLocation(tokLin, tokCol, tokELin, tokECol);
|
|
// End, user-specified epilog
|
|
} // end finally
|
|
}
|
|
|
|
#if BACKUP
|
|
Result Recurse2(ref Context ctx, int next)
|
|
{
|
|
// Assert: at entry "state" is an accept state AND
|
|
// NextState(state, code) != goStart AND
|
|
// NextState(state, code) is not an accept state.
|
|
//
|
|
SaveStateAndPos(ref ctx);
|
|
state = next;
|
|
GetCode();
|
|
|
|
while ((next = NextState()) > eofNum)
|
|
{
|
|
if (state <= maxAccept && next > maxAccept) // need to update backup data
|
|
SaveStateAndPos(ref ctx);
|
|
state = next;
|
|
if (state == eofNum) return Result.accept;
|
|
GetCode();
|
|
}
|
|
return (state <= maxAccept ? Result.accept : Result.noMatch);
|
|
}
|
|
|
|
void SaveStateAndPos(ref Context ctx)
|
|
{
|
|
ctx.bPos = buffer.Pos;
|
|
ctx.rPos = readPos;
|
|
ctx.cCol = cCol;
|
|
ctx.lNum = lNum;
|
|
ctx.state = state;
|
|
ctx.cChr = code;
|
|
}
|
|
|
|
void RestoreStateAndPos(ref Context ctx)
|
|
{
|
|
buffer.Pos = ctx.bPos;
|
|
readPos = ctx.rPos;
|
|
cCol = ctx.cCol;
|
|
lNum = ctx.lNum;
|
|
state = ctx.state;
|
|
code = ctx.cChr;
|
|
}
|
|
|
|
#endif // BACKUP
|
|
|
|
// ============= End of the tokenizer code ================
|
|
|
|
#if STACK
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
internal void yy_clear_stack() { scStack.Clear(); }
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
internal int yy_top_state() { return scStack.Peek(); }
|
|
|
|
internal void yy_push_state(int state)
|
|
{
|
|
scStack.Push(currentScOrd);
|
|
BEGIN(state);
|
|
}
|
|
|
|
internal void yy_pop_state()
|
|
{
|
|
// Protect against input errors that pop too far ...
|
|
if (scStack.Count > 0) {
|
|
int newSc = scStack.Pop();
|
|
BEGIN(newSc);
|
|
} // Otherwise leave stack unchanged.
|
|
}
|
|
#endif // STACK
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
|
|
internal void ECHO() { Console.Out.Write(yytext); }
|
|
|
|
#region UserCodeSection
|
|
|
|
static int mlCommentCnt = 0;
|
|
|
|
public override void yyerror(string format, params object[] args)
|
|
{
|
|
string errorMsg = PT.CreateErrorString(args);
|
|
PT.AddError(errorMsg,yylloc);
|
|
}
|
|
|
|
// Статический класс, определяющий ключевые слова языка
|
|
public static class Keywords
|
|
{
|
|
private static Dictionary<string, int> keywords = new Dictionary<string, int>();
|
|
|
|
static Keywords()
|
|
{
|
|
keywords.Add("TRUE", (int)Tokens.TRUE);
|
|
keywords.Add("FALSE", (int)Tokens.FALSE);
|
|
keywords.Add("ODD", (int)Tokens.ODD);
|
|
keywords.Add("OR", (int)Tokens.OR);
|
|
//keywords.Add("DIV", (int)Tokens.DIV);
|
|
//keywords.Add("MOD", (int)Tokens.MOD);
|
|
keywords.Add("IF", (int)Tokens.IF);
|
|
keywords.Add("THEN", (int)Tokens.THEN);
|
|
keywords.Add("ELSE", (int)Tokens.ELSE);
|
|
keywords.Add("BEGIN", (int)Tokens.BEGIN);
|
|
keywords.Add("END", (int)Tokens.END);
|
|
keywords.Add("WHILE", (int)Tokens.WHILE);
|
|
keywords.Add("DO", (int)Tokens.DO);
|
|
keywords.Add("MODULE", (int)Tokens.MODULE);
|
|
keywords.Add("CONST", (int)Tokens.CONST);
|
|
keywords.Add("VAR", (int)Tokens.VAR);
|
|
keywords.Add("PROCEDURE", (int)Tokens.PROCEDURE);
|
|
}
|
|
public static int KeywordOrIDToken(string s)
|
|
{
|
|
if (keywords.ContainsKey(s))
|
|
return keywords[s];
|
|
else
|
|
return (int)Tokens.ID;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
} // end class $Scanner
|
|
|
|
// ==============================================================
|
|
// <auto-generated>
|
|
// This code automatically produced from an embedded resource.
|
|
// Do not edit this file, or it will become incompatible with
|
|
// the specification from which it was generated.
|
|
// </auto-generated>
|
|
// ==============================================================
|
|
|
|
// Code copied from GPLEX embedded resource
|
|
[Serializable]
|
|
public class BufferException : Exception
|
|
{
|
|
public BufferException() { }
|
|
public BufferException(string message) : base(message) { }
|
|
public BufferException(string message, Exception innerException)
|
|
: base(message, innerException) { }
|
|
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
: base(info, context) { }
|
|
}
|
|
|
|
public abstract class ScanBuff
|
|
{
|
|
private string fileNm;
|
|
|
|
public const int EndOfFile = -1;
|
|
public const int UnicodeReplacementChar = 0xFFFD;
|
|
|
|
public bool IsFile { get { return (fileNm != null); } }
|
|
public string FileName { get { return fileNm; } set { fileNm = value; } }
|
|
|
|
public abstract int Pos { get; set; }
|
|
public abstract int Read();
|
|
public virtual void Mark() { }
|
|
|
|
public abstract string GetString(int begin, int limit);
|
|
|
|
public static ScanBuff GetBuffer(string source)
|
|
{
|
|
return new StringBuffer(source);
|
|
}
|
|
|
|
public static ScanBuff GetBuffer(IList<string> source)
|
|
{
|
|
return new LineBuffer(source);
|
|
}
|
|
|
|
public static ScanBuff GetBuffer(Stream source)
|
|
{
|
|
return new BuildBuffer(source);
|
|
}
|
|
|
|
#if (!BYTEMODE)
|
|
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
|
{
|
|
return new BuildBuffer(source, fallbackCodePage);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
#region Buffer classes
|
|
|
|
// ==============================================================
|
|
// ===== Definitions for various ScanBuff derived classes ====
|
|
// ==============================================================
|
|
// =============== String input ================
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This class reads characters from a single string as
|
|
/// required, for example, by Visual Studio language services
|
|
/// </summary>
|
|
sealed class StringBuffer : ScanBuff
|
|
{
|
|
string str; // input buffer
|
|
int bPos; // current position in buffer
|
|
int sLen;
|
|
|
|
public StringBuffer(string source)
|
|
{
|
|
this.str = source;
|
|
this.sLen = source.Length;
|
|
this.FileName = null;
|
|
}
|
|
|
|
public override int Read()
|
|
{
|
|
if (bPos < sLen) return str[bPos++];
|
|
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
|
|
else { bPos++; return EndOfFile; } // two strikes and you're out!
|
|
}
|
|
|
|
public override string GetString(int begin, int limit)
|
|
{
|
|
// "limit" can be greater than sLen with the BABEL
|
|
// option set. Read returns a "virtual" EOL if
|
|
// an attempt is made to read past the end of the
|
|
// string buffer. Without the guard any attempt
|
|
// to fetch yytext for a token that includes the
|
|
// EOL will throw an index exception.
|
|
if (limit > sLen) limit = sLen;
|
|
if (limit <= begin) return "";
|
|
else return str.Substring(begin, limit - begin);
|
|
}
|
|
|
|
public override int Pos
|
|
{
|
|
get { return bPos; }
|
|
set { bPos = value; }
|
|
}
|
|
|
|
public override string ToString() { return "StringBuffer"; }
|
|
}
|
|
|
|
// ==============================================================
|
|
// The LineBuff class contributed by Nigel Horspool,
|
|
// nigelh@cs.uvic.cs
|
|
// ==============================================================
|
|
|
|
sealed class LineBuffer : ScanBuff
|
|
{
|
|
IList<string> line; // list of source lines from a file
|
|
int numLines; // number of strings in line list
|
|
string curLine; // current line in that list
|
|
int cLine; // index of current line in the list
|
|
int curLen; // length of current line
|
|
int curLineStart; // position of line start in whole file
|
|
int curLineEnd; // position of line end in whole file
|
|
int maxPos; // max position ever visited in whole file
|
|
int cPos; // ordinal number of code in source
|
|
|
|
// Constructed from a list of strings, one per source line.
|
|
// The lines have had trailing '\n' characters removed.
|
|
public LineBuffer(IList<string> lineList)
|
|
{
|
|
line = lineList;
|
|
numLines = line.Count;
|
|
cPos = curLineStart = 0;
|
|
curLine = (numLines > 0 ? line[0] : "");
|
|
maxPos = curLineEnd = curLen = curLine.Length;
|
|
cLine = 1;
|
|
FileName = null;
|
|
}
|
|
|
|
public override int Read()
|
|
{
|
|
if (cPos < curLineEnd)
|
|
return curLine[cPos++ - curLineStart];
|
|
if (cPos++ == curLineEnd)
|
|
return '\n';
|
|
if (cLine >= numLines)
|
|
return EndOfFile;
|
|
curLine = line[cLine];
|
|
curLen = curLine.Length;
|
|
curLineStart = curLineEnd + 1;
|
|
curLineEnd = curLineStart + curLen;
|
|
if (curLineEnd > maxPos)
|
|
maxPos = curLineEnd;
|
|
cLine++;
|
|
return curLen > 0 ? curLine[0] : '\n';
|
|
}
|
|
|
|
// To speed up searches for the line containing a position
|
|
private int cachedPosition;
|
|
private int cachedIxdex;
|
|
private int cachedLineStart;
|
|
|
|
// Given a position pos within the entire source, the results are
|
|
// ix -- the index of the containing line
|
|
// lstart -- the position of the first character on that line
|
|
private void findIndex(int pos, out int ix, out int lstart)
|
|
{
|
|
if (pos >= cachedPosition)
|
|
{
|
|
ix = cachedIxdex; lstart = cachedLineStart;
|
|
}
|
|
else
|
|
{
|
|
ix = lstart = 0;
|
|
}
|
|
for (; ; )
|
|
{
|
|
int len = line[ix].Length + 1;
|
|
if (pos < lstart + len) break;
|
|
lstart += len;
|
|
ix++;
|
|
}
|
|
cachedPosition = pos;
|
|
cachedIxdex = ix;
|
|
cachedLineStart = lstart;
|
|
}
|
|
|
|
public override string GetString(int begin, int limit)
|
|
{
|
|
if (begin >= maxPos || limit <= begin) return "";
|
|
int endIx, begIx, endLineStart, begLineStart;
|
|
findIndex(begin, out begIx, out begLineStart);
|
|
int begCol = begin - begLineStart;
|
|
findIndex(limit, out endIx, out endLineStart);
|
|
int endCol = limit - endLineStart;
|
|
string s = line[begIx];
|
|
if (begIx == endIx)
|
|
{
|
|
// the usual case, substring all on one line
|
|
return (endCol <= s.Length) ?
|
|
s.Substring(begCol, endCol - begCol)
|
|
: s.Substring(begCol) + "\n";
|
|
}
|
|
// the string spans multiple lines, yuk!
|
|
StringBuilder sb = new StringBuilder();
|
|
if (begCol < s.Length)
|
|
sb.Append(s.Substring(begCol));
|
|
for (; ; )
|
|
{
|
|
sb.Append("\n");
|
|
s = line[++begIx];
|
|
if (begIx >= endIx) break;
|
|
sb.Append(s);
|
|
}
|
|
if (endCol <= s.Length)
|
|
{
|
|
sb.Append(s.Substring(0, endCol));
|
|
}
|
|
else
|
|
{
|
|
sb.Append(s);
|
|
sb.Append("\n");
|
|
}
|
|
return sb.ToString();
|
|
}
|
|
|
|
public override int Pos
|
|
{
|
|
get { return cPos; }
|
|
set
|
|
{
|
|
cPos = value;
|
|
findIndex(cPos, out cLine, out curLineStart);
|
|
curLine = line[cLine];
|
|
curLineEnd = curLineStart + curLine.Length;
|
|
}
|
|
}
|
|
|
|
public override string ToString() { return "LineBuffer"; }
|
|
}
|
|
|
|
|
|
// ==============================================================
|
|
// ===== class BuildBuff : for unicode text files ========
|
|
// ==============================================================
|
|
|
|
class BuildBuffer : ScanBuff
|
|
{
|
|
// Double buffer for char stream.
|
|
class BufferElement
|
|
{
|
|
StringBuilder bldr = new StringBuilder();
|
|
StringBuilder next = new StringBuilder();
|
|
int minIx;
|
|
int maxIx;
|
|
int brkIx;
|
|
bool appendToNext;
|
|
|
|
internal BufferElement() { }
|
|
|
|
internal int MaxIndex { get { return maxIx; } }
|
|
// internal int MinIndex { get { return minIx; } }
|
|
|
|
internal char this[int index]
|
|
{
|
|
get
|
|
{
|
|
if (index < minIx || index >= maxIx)
|
|
throw new BufferException("Index was outside data buffer");
|
|
else if (index < brkIx)
|
|
return bldr[index - minIx];
|
|
else
|
|
return next[index - brkIx];
|
|
}
|
|
}
|
|
|
|
internal void Append(char[] block, int count)
|
|
{
|
|
maxIx += count;
|
|
if (appendToNext)
|
|
this.next.Append(block, 0, count);
|
|
else
|
|
{
|
|
this.bldr.Append(block, 0, count);
|
|
brkIx = maxIx;
|
|
appendToNext = true;
|
|
}
|
|
}
|
|
|
|
internal string GetString(int start, int limit)
|
|
{
|
|
if (limit <= start)
|
|
return "";
|
|
if (start >= minIx && limit <= maxIx)
|
|
if (limit < brkIx) // String entirely in bldr builder
|
|
return bldr.ToString(start - minIx, limit - start);
|
|
else if (start >= brkIx) // String entirely in next builder
|
|
return next.ToString(start - brkIx, limit - start);
|
|
else // Must do a string-concatenation
|
|
return
|
|
bldr.ToString(start - minIx, brkIx - start) +
|
|
next.ToString(0, limit - brkIx);
|
|
else
|
|
throw new BufferException("String was outside data buffer");
|
|
}
|
|
|
|
internal void Mark(int limit)
|
|
{
|
|
if (limit > brkIx + 16) // Rotate blocks
|
|
{
|
|
StringBuilder temp = bldr;
|
|
bldr = next;
|
|
next = temp;
|
|
next.Length = 0;
|
|
minIx = brkIx;
|
|
brkIx = maxIx;
|
|
}
|
|
}
|
|
}
|
|
|
|
BufferElement data = new BufferElement();
|
|
|
|
int bPos; // Postion index in the StringBuilder
|
|
BlockReader NextBlk; // Delegate that serves char-arrays;
|
|
|
|
private string EncodingName
|
|
{
|
|
get
|
|
{
|
|
StreamReader rdr = NextBlk.Target as StreamReader;
|
|
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
|
|
}
|
|
}
|
|
|
|
public BuildBuffer(Stream stream)
|
|
{
|
|
FileStream fStrm = (stream as FileStream);
|
|
if (fStrm != null) FileName = fStrm.Name;
|
|
NextBlk = BlockReaderFactory.Raw(stream);
|
|
}
|
|
|
|
#if (!BYTEMODE)
|
|
public BuildBuffer(Stream stream, int fallbackCodePage)
|
|
{
|
|
FileStream fStrm = (stream as FileStream);
|
|
if (fStrm != null) FileName = fStrm.Name;
|
|
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
|
}
|
|
#endif
|
|
|
|
/// <summary>
|
|
/// Marks a conservative lower bound for the buffer,
|
|
/// allowing space to be reclaimed. If an application
|
|
/// needs to call GetString at arbitrary past locations
|
|
/// in the input stream, Mark() is not called.
|
|
/// </summary>
|
|
public override void Mark() { data.Mark(bPos - 2); }
|
|
|
|
public override int Pos
|
|
{
|
|
get { return bPos; }
|
|
set { bPos = value; }
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Read returns the ordinal number of the next char, or
|
|
/// EOF (-1) for an end of stream. Note that the next
|
|
/// code point may require *two* calls of Read().
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public override int Read()
|
|
{
|
|
//
|
|
// Characters at positions
|
|
// [data.offset, data.offset + data.bldr.Length)
|
|
// are available in data.bldr.
|
|
//
|
|
if (bPos < data.MaxIndex)
|
|
{
|
|
// ch0 cannot be EOF
|
|
return (int)data[bPos++];
|
|
}
|
|
else // Read from underlying stream
|
|
{
|
|
// Experimental code, blocks of page size
|
|
char[] chrs = new char[4096];
|
|
int count = NextBlk(chrs, 0, 4096);
|
|
if (count == 0)
|
|
return EndOfFile;
|
|
else
|
|
{
|
|
data.Append(chrs, count);
|
|
return (int)data[bPos++];
|
|
}
|
|
}
|
|
}
|
|
|
|
public override string GetString(int begin, int limit)
|
|
{
|
|
return data.GetString(begin, limit);
|
|
}
|
|
|
|
public override string ToString()
|
|
{
|
|
return "StringBuilder buffer, encoding: " + this.EncodingName;
|
|
}
|
|
}
|
|
|
|
// =============== End ScanBuff-derived classes ==================
|
|
|
|
public delegate int BlockReader(char[] block, int index, int number);
|
|
|
|
// A delegate factory, serving up a delegate that
|
|
// reads a block of characters from the underlying
|
|
// encoded stream, via a StreamReader object.
|
|
//
|
|
public static class BlockReaderFactory
|
|
{
|
|
public static BlockReader Raw(Stream stream)
|
|
{
|
|
return delegate(char[] block, int index, int number)
|
|
{
|
|
byte[] b = new byte[number];
|
|
int count = stream.Read(b, 0, number);
|
|
int i = 0;
|
|
int j = index;
|
|
for (; i < count; i++, j++)
|
|
block[j] = (char)b[i];
|
|
return count;
|
|
};
|
|
}
|
|
|
|
#if (!BYTEMODE)
|
|
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
{
|
|
Encoding encoding;
|
|
int preamble = Preamble(stream);
|
|
|
|
if (preamble != 0) // There is a valid BOM here!
|
|
encoding = Encoding.GetEncoding(preamble);
|
|
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
|
return Raw(stream);
|
|
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
encoding = Encoding.GetEncoding(fallbackCodePage);
|
|
else // This is the "guess" option
|
|
{
|
|
int guess = new Guesser(stream).GuessCodePage();
|
|
stream.Seek(0, SeekOrigin.Begin);
|
|
if (guess == -1) // ==> this is a 7-bit file
|
|
encoding = Encoding.ASCII;
|
|
else if (guess == 65001)
|
|
encoding = Encoding.UTF8;
|
|
else // ==> use the machine default
|
|
encoding = Encoding.Default;
|
|
}
|
|
StreamReader reader = new StreamReader(stream, encoding);
|
|
return reader.Read;
|
|
}
|
|
|
|
static int Preamble(Stream stream)
|
|
{
|
|
int b0 = stream.ReadByte();
|
|
int b1 = stream.ReadByte();
|
|
|
|
if (b0 == 0xfe && b1 == 0xff)
|
|
return 1201; // UTF16BE
|
|
if (b0 == 0xff && b1 == 0xfe)
|
|
return 1200; // UTF16LE
|
|
|
|
int b2 = stream.ReadByte();
|
|
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
|
return 65001; // UTF8
|
|
//
|
|
// There is no unicode preamble, so we
|
|
// return denoter for the machine default.
|
|
//
|
|
stream.Seek(0, SeekOrigin.Begin);
|
|
return 0;
|
|
}
|
|
#endif // !BYTEMODE
|
|
}
|
|
#endregion Buffer classes
|
|
|
|
// ==============================================================
|
|
// ============ class CodePageHandling =============
|
|
// ==============================================================
|
|
|
|
public static class CodePageHandling
|
|
{
|
|
public static int GetCodePage(string option)
|
|
{
|
|
string command = option.ToUpperInvariant();
|
|
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
|
|
command = command.Substring(9);
|
|
try
|
|
{
|
|
if (command.Equals("RAW"))
|
|
return -1;
|
|
else if (command.Equals("GUESS"))
|
|
return -2;
|
|
else if (command.Equals("DEFAULT"))
|
|
return 0;
|
|
else if (char.IsDigit(command[0]))
|
|
return int.Parse(command, CultureInfo.InvariantCulture);
|
|
else
|
|
{
|
|
Encoding enc = Encoding.GetEncoding(command);
|
|
return enc.CodePage;
|
|
}
|
|
}
|
|
catch (FormatException)
|
|
{
|
|
Console.Error.WriteLine(
|
|
"Invalid format \"{0}\", using machine default", option);
|
|
}
|
|
catch (ArgumentException)
|
|
{
|
|
Console.Error.WriteLine(
|
|
"Unknown code page \"{0}\", using machine default", option);
|
|
}
|
|
return 0;
|
|
}
|
|
}
|
|
#region guesser
|
|
#if (!BYTEMODE)
|
|
// ==============================================================
|
|
// ============ Encoding Guesser =============
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This class provides a simple finite state automaton that
|
|
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
/// The method then guesses whether it is UTF-8 or maybe some
|
|
/// local machine default encoding. This works well for the
|
|
/// various Latin encodings.
|
|
/// </summary>
|
|
internal class Guesser
|
|
{
|
|
ScanBuff buffer;
|
|
|
|
public int GuessCodePage() { return Scan(); }
|
|
|
|
const int maxAccept = 10;
|
|
const int initial = 0;
|
|
const int eofNum = 0;
|
|
const int goStart = -1;
|
|
const int INITIAL = 0;
|
|
const int EndToken = 0;
|
|
|
|
#region user code
|
|
/*
|
|
* Reads the bytes of a file to determine if it is
|
|
* UTF-8 or a single-byte code page file.
|
|
*/
|
|
public long utfX;
|
|
public long uppr;
|
|
#endregion user code
|
|
|
|
int state;
|
|
int currentStart = startState[0];
|
|
int code;
|
|
|
|
#region ScannerTables
|
|
static int[] startState = new int[] { 11, 0 };
|
|
|
|
#region CharacterMap
|
|
static sbyte[] map = new sbyte[256] {
|
|
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
#endregion
|
|
|
|
static sbyte[][] nextState = new sbyte[][] {
|
|
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
null,
|
|
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
null,
|
|
null,
|
|
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
};
|
|
|
|
|
|
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
// Reason for suppression: cannot have self-reference in array initializer.
|
|
static Guesser()
|
|
{
|
|
nextState[7] = nextState[2];
|
|
nextState[9] = nextState[2];
|
|
nextState[10] = nextState[2];
|
|
}
|
|
|
|
int NextState()
|
|
{
|
|
if (code == ScanBuff.EndOfFile)
|
|
return eofNum;
|
|
else
|
|
return nextState[state][map[code]];
|
|
}
|
|
#endregion
|
|
|
|
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
|
|
public void SetSource(System.IO.Stream source)
|
|
{
|
|
this.buffer = new BuildBuffer(source);
|
|
code = buffer.Read();
|
|
}
|
|
|
|
int Scan()
|
|
{
|
|
for (; ; )
|
|
{
|
|
int next;
|
|
state = currentStart;
|
|
while ((next = NextState()) == goStart)
|
|
code = buffer.Read();
|
|
|
|
state = next;
|
|
code = buffer.Read();
|
|
|
|
while ((next = NextState()) > eofNum)
|
|
{
|
|
state = next;
|
|
code = buffer.Read();
|
|
}
|
|
if (state <= maxAccept)
|
|
{
|
|
#region ActionSwitch
|
|
#pragma warning disable 162
|
|
switch (state)
|
|
{
|
|
case eofNum:
|
|
switch (currentStart)
|
|
{
|
|
case 11:
|
|
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
else return 65001; /* UTF-8 encoding */
|
|
break;
|
|
}
|
|
return EndToken;
|
|
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
uppr++;
|
|
break;
|
|
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
uppr += 2;
|
|
break;
|
|
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
uppr += 3;
|
|
break;
|
|
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
utfX += 3;
|
|
break;
|
|
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
uppr += 2;
|
|
break;
|
|
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
utfX += 2;
|
|
break;
|
|
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
utfX++;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
#pragma warning restore 162
|
|
#endregion
|
|
}
|
|
}
|
|
}
|
|
} // end class Guesser
|
|
|
|
#endif // !BYTEMODE
|
|
#endregion
|
|
|
|
// End of code copied from embedded resource
|
|
|
|
} // end namespace
|