1717 lines
63 KiB
C#
1717 lines
63 KiB
C#
//
|
|
// This CSharp output file generated by Gardens Point LEX
|
|
// Version: 1.0.1.250 (2009-01-01)
|
|
// Machine: PC
|
|
// DateTime: 30.06.2009 22:50:21
|
|
// UserName: A
|
|
// GPLEX input file <python_lex.lex>
|
|
// GPLEX frame file <gplexx.frame>
|
|
//
|
|
// Option settings: parser, minimize, compressnext
|
|
//
|
|
|
|
//
|
|
// gplexx.frame
|
|
// Version 1.0.0 of 01-November-2008
|
|
//
|
|
#define BACKUP
|
|
#define BYTEMODE
|
|
|
|
using System;
|
|
using System.IO;
|
|
using System.Text;
|
|
using System.Collections.Generic;
|
|
#if !STANDALONE
|
|
using gppg;
|
|
#endif
|
|
using PascalABCCompiler.SyntaxTree;
|
|
using System.Globalization;
|
|
using GPPGTools;
|
|
|
|
namespace PascalABCCompiler.PythonABCParser
|
|
{
|
|
/// <summary>
|
|
/// Summary Canonical example of GPLEX automaton
|
|
/// </summary>
|
|
|
|
#if STANDALONE
|
|
//
|
|
// These are the dummy declarations for stand-alone GPLEX applications
|
|
// normally these declarations would come from the parser.
|
|
// If you declare /noparser, or %option noparser then you get this.
|
|
//
|
|
|
|
public enum Tokens
|
|
{
|
|
EOF = 0, maxParseToken = int.MaxValue
|
|
// must have at least these two, values are almost arbitrary
|
|
}
|
|
|
|
public abstract class ScanBase
|
|
{
|
|
public abstract int yylex();
|
|
protected virtual bool yywrap() { return true; }
|
|
#if BABEL
|
|
protected abstract int CurrentSc { get; set; }
|
|
// EolState is the 32-bit of state data persisted at
|
|
// the end of each line for Visual Studio colorization.
|
|
// The default is to return CurrentSc. You must override
|
|
// this if you want more complicated behavior.
|
|
public virtual int EolState {
|
|
get { return CurrentSc; }
|
|
set { CurrentSc = value; }
|
|
}
|
|
}
|
|
|
|
public interface IColorScan
|
|
{
|
|
void SetSource(string source, int offset);
|
|
int GetNext(ref int state, out int start, out int end);
|
|
#endif // BABEL
|
|
}
|
|
|
|
#endif // STANDALONE
|
|
|
|
public abstract class ScanBuff
|
|
{
|
|
public const int EOF = -1;
|
|
public abstract int Pos { get; set; }
|
|
public abstract int Read();
|
|
public abstract int Peek();
|
|
public abstract int ReadPos { get; }
|
|
public abstract string GetString(int b, int e);
|
|
|
|
public const int UnicodeReplacementChar = 0xFFFD;
|
|
}
|
|
|
|
// If the compiler can't find ScanBase maybe you need to run
|
|
// GPPG with the /gplex option, or GPLEX with /noparser
|
|
#if BABEL
|
|
public sealed partial class Scanner : ScanBase, IColorScan
|
|
{
|
|
public ScanBuff buffer;
|
|
int currentScOrd = 0; // start condition ordinal
|
|
|
|
protected override int CurrentSc
|
|
{
|
|
// The current start state is a property
|
|
// to try to avoid the user error of setting
|
|
// scState but forgetting to update the FSA
|
|
// start state "currentStart"
|
|
//
|
|
get { return currentScOrd; } // i.e. return YY_START;
|
|
set { currentScOrd = value; // i.e. BEGIN(value);
|
|
currentStart = startState[value]; }
|
|
}
|
|
#else // BABEL
|
|
public sealed partial class Scanner : ScanBase
|
|
{
|
|
public ScanBuff buffer;
|
|
int currentScOrd = 0; // start condition ordinal
|
|
#endif // BABEL
|
|
|
|
private static int GetMaxParseToken() {
|
|
System.Reflection.FieldInfo f = typeof(Tokens).GetField("maxParseToken");
|
|
return (f == null ? int.MaxValue : (int)f.GetValue(null));
|
|
}
|
|
|
|
static int parserMax = GetMaxParseToken();
|
|
|
|
enum Result {accept, noMatch, contextFound};
|
|
|
|
const int maxAccept = 49;
|
|
const int initial = 50;
|
|
const int eofNum = 0;
|
|
const int goStart = -1;
|
|
const int INITIAL = 0;
|
|
|
|
#region user code
|
|
PythonParserTools PT = new PythonParserTools();
|
|
#endregion user code
|
|
|
|
int state;
|
|
int currentStart = startState[0];
|
|
int chr; // last character read
|
|
int cNum = 0; // ordinal number of chr
|
|
int lNum = 0; // current line number
|
|
int lineStartNum; // ordinal number at start of line
|
|
//
|
|
// The following instance variables are used, among other
|
|
// things, for constructing the yylloc location objects.
|
|
//
|
|
int tokPos; // buffer position at start of token
|
|
int tokNum; // ordinal number of first character
|
|
int tokLen; // number of characters in token
|
|
int tokCol; // zero-based column number at start of token
|
|
int tokLin; // line number at start of token
|
|
int tokEPos; // buffer position at end of token
|
|
int tokECol; // column number at end of token
|
|
int tokELin; // line number at end of token
|
|
string tokTxt; // lazily constructed text of token
|
|
#if STACK
|
|
private Stack<int> scStack = new Stack<int>();
|
|
#endif // STACK
|
|
|
|
#region ScannerTables
|
|
struct Table {
|
|
public int min; public int rng; public int dflt;
|
|
public sbyte[] nxt;
|
|
public Table(int m, int x, int d, sbyte[] n) {
|
|
min = m; rng = x; dflt = d; nxt = n;
|
|
}
|
|
};
|
|
|
|
static int[] startState = {50, 0};
|
|
|
|
static Table[] NxS = new Table[64];
|
|
|
|
static Scanner() {
|
|
NxS[0] = new Table(0, 0, 0, null);
|
|
NxS[1] = new Table(0, 0, -1, null);
|
|
NxS[2] = new Table(0, 0, -1, null);
|
|
NxS[3] = new Table(10, 1, -1, new sbyte[] {2});
|
|
NxS[4] = new Table(0, 0, -1, null);
|
|
NxS[5] = new Table(0, 0, -1, null);
|
|
NxS[6] = new Table(0, 0, -1, null);
|
|
NxS[7] = new Table(0, 0, -1, null);
|
|
NxS[8] = new Table(0, 0, -1, null);
|
|
NxS[9] = new Table(62, 1, -1, new sbyte[] {42});
|
|
NxS[10] = new Table(0, 0, -1, null);
|
|
NxS[11] = new Table(0, 0, -1, null);
|
|
NxS[12] = new Table(46, 12, -1, new sbyte[] {55, -1, 12, 12, 12, 12,
|
|
12, 12, 12, 12, 12, 12});
|
|
NxS[13] = new Table(0, 0, -1, null);
|
|
NxS[14] = new Table(0, 0, -1, null);
|
|
NxS[15] = new Table(61, 1, -1, new sbyte[] {40});
|
|
NxS[16] = new Table(61, 1, -1, new sbyte[] {39});
|
|
NxS[17] = new Table(61, 1, -1, new sbyte[] {38});
|
|
NxS[18] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[19] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 34, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[20] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 31, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[21] = new Table(0, 0, -1, null);
|
|
NxS[22] = new Table(0, 0, -1, null);
|
|
NxS[23] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 29, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[24] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 27, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[25] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 26, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[26] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[27] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 28, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[28] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[29] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 30, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[30] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[31] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 32,
|
|
18, 18, 18, 18, 18});
|
|
NxS[32] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 33,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[33] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[34] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 35, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[35] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 36, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[36] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 37,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[37] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18});
|
|
NxS[38] = new Table(0, 0, -1, null);
|
|
NxS[39] = new Table(0, 0, -1, null);
|
|
NxS[40] = new Table(0, 0, -1, null);
|
|
NxS[41] = new Table(48, 10, -1, new sbyte[] {41, 41, 41, 41, 41, 41,
|
|
41, 41, 41, 41});
|
|
NxS[42] = new Table(0, 0, -1, null);
|
|
NxS[43] = new Table(39, 1, -1, new sbyte[] {57});
|
|
NxS[44] = new Table(0, 0, -1, null);
|
|
NxS[45] = new Table(0, 0, -1, null);
|
|
NxS[46] = new Table(0, 0, -1, null);
|
|
NxS[47] = new Table(10, 4, 53, new sbyte[] {46, 53, 53, 47});
|
|
NxS[48] = new Table(34, 1, -1, new sbyte[] {61});
|
|
NxS[49] = new Table(0, 0, -1, null);
|
|
NxS[50] = new Table(9, 114, -1, new sbyte[] {1, 2, -1, -1, 3, -1,
|
|
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
|
-1, 1, 51, 52, 53, -1, -1, -1, 54, 4, 5, 6, 7, 8, 9, 10,
|
|
11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 16, 17,
|
|
-1, -1, 18, 18, 18, 18, 18, 19, 18, 18, 18, 18, 18, 18, 18, 18,
|
|
18, 18, 18, 18, 18, 20, 18, 18, 18, 18, 18, 18, 21, -1, 22, -1,
|
|
18, -1, 23, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 24,
|
|
25, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18});
|
|
NxS[51] = new Table(61, 1, -1, new sbyte[] {49});
|
|
NxS[52] = new Table(10, 25, 60, new sbyte[] {-1, 60, 60, 60, 60, 60,
|
|
60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60,
|
|
60, 60, 48});
|
|
NxS[53] = new Table(10, 4, 53, new sbyte[] {46, 53, 53, 47});
|
|
NxS[54] = new Table(10, 30, 56, new sbyte[] {-1, 56, 56, 56, 56, 56,
|
|
56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
|
|
56, 56, 56, 56, 56, 56, 56, 43});
|
|
NxS[55] = new Table(48, 10, -1, new sbyte[] {41, 41, 41, 41, 41, 41,
|
|
41, 41, 41, 41});
|
|
NxS[56] = new Table(10, 30, 56, new sbyte[] {-1, 56, 56, 56, 56, 56,
|
|
56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
|
|
56, 56, 56, 56, 56, 56, 56, 45});
|
|
NxS[57] = new Table(39, 1, 57, new sbyte[] {58});
|
|
NxS[58] = new Table(39, 1, -1, new sbyte[] {59});
|
|
NxS[59] = new Table(39, 1, -1, new sbyte[] {44});
|
|
NxS[60] = new Table(10, 25, 60, new sbyte[] {-1, 60, 60, 60, 60, 60,
|
|
60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60,
|
|
60, 60, 45});
|
|
NxS[61] = new Table(34, 1, 61, new sbyte[] {62});
|
|
NxS[62] = new Table(34, 1, -1, new sbyte[] {63});
|
|
NxS[63] = new Table(34, 1, -1, new sbyte[] {44});
|
|
}
|
|
|
|
int NextState(int qStat) {
|
|
if (chr == ScanBuff.EOF)
|
|
return eofNum;
|
|
else {
|
|
int rslt;
|
|
int idx = (byte)(chr - NxS[qStat].min);
|
|
if ((uint)idx >= (uint)NxS[qStat].rng) rslt = NxS[qStat].dflt;
|
|
else rslt = NxS[qStat].nxt[idx];
|
|
return rslt;
|
|
}
|
|
}
|
|
|
|
int NextState() {
|
|
if (chr == ScanBuff.EOF)
|
|
return eofNum;
|
|
else {
|
|
int rslt;
|
|
int idx = (byte)(chr - NxS[state].min);
|
|
if ((uint)idx >= (uint)NxS[state].rng) rslt = NxS[state].dflt;
|
|
else rslt = NxS[state].nxt[idx];
|
|
return rslt;
|
|
}
|
|
}
|
|
|
|
// EXPERIMENTAL: This is the NextState method that
|
|
// is used by the CharClassPredicate functions for V0.9
|
|
int TestNextState(int qStat, int chr) {
|
|
if (chr == ScanBuff.EOF)
|
|
return eofNum;
|
|
else {
|
|
int rslt;
|
|
int idx = (byte)(chr - NxS[qStat].min);
|
|
if ((uint)idx >= (uint)NxS[qStat].rng) rslt = NxS[qStat].dflt;
|
|
else rslt = NxS[qStat].nxt[idx];
|
|
return rslt;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
|
|
#if BACKUP
|
|
// ==============================================================
|
|
// == Nested class used for backup in automata which do backup ==
|
|
// ==============================================================
|
|
|
|
internal class Context // class used for automaton backup.
|
|
{
|
|
public int bPos;
|
|
public int cNum;
|
|
public int lNum; // Need this in case of backup over EOL.
|
|
public int state;
|
|
public int cChr;
|
|
}
|
|
#endif // BACKUP
|
|
|
|
// ==============================================================
|
|
// ==== Nested struct to support input switching in scanners ====
|
|
// ==============================================================
|
|
|
|
struct BufferContext {
|
|
internal ScanBuff buffSv;
|
|
internal int chrSv;
|
|
internal int cNumSv;
|
|
internal int lNumSv;
|
|
internal int startSv;
|
|
}
|
|
|
|
#region Buffer classes
|
|
|
|
// ==============================================================
|
|
// ===== Private methods to save and restore buffer contexts ====
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This method creates a buffer context record from
|
|
/// the current buffer object, together with some
|
|
/// scanner state values.
|
|
/// </summary>
|
|
BufferContext MkBuffCtx()
|
|
{
|
|
BufferContext rslt;
|
|
rslt.buffSv = this.buffer;
|
|
rslt.chrSv = this.chr;
|
|
rslt.cNumSv = this.cNum;
|
|
rslt.lNumSv = this.lNum;
|
|
rslt.startSv = this.lineStartNum;
|
|
return rslt;
|
|
}
|
|
|
|
/// <summary>
|
|
/// This method restores the buffer value and allied
|
|
/// scanner state from the given context record value.
|
|
/// </summary>
|
|
void RestoreBuffCtx(BufferContext value)
|
|
{
|
|
this.buffer = value.buffSv;
|
|
this.chr = value.chrSv;
|
|
this.cNum = value.cNumSv;
|
|
this.lNum = value.lNumSv;
|
|
this.lineStartNum = value.startSv;
|
|
}
|
|
|
|
// ==============================================================
|
|
// ===== Nested classes for various ScanBuff derived classes ====
|
|
// ==============================================================
|
|
// =============== Nested class for string input ================
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This class read characters from a single string as
|
|
/// required, for example, by Visual Studio language services
|
|
/// </summary>
|
|
public sealed class StringBuff : ScanBuff
|
|
{
|
|
string str; // input buffer
|
|
int bPos; // current position in buffer
|
|
int sLen;
|
|
|
|
public StringBuff(string str)
|
|
{
|
|
this.str = str;
|
|
this.sLen = str.Length;
|
|
}
|
|
|
|
private int Read16()
|
|
{
|
|
if (bPos < sLen) return str[bPos++];
|
|
#if BABEL
|
|
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see newline
|
|
#endif // BABEL
|
|
else { bPos++; return EOF; } // two strikes and you're out!
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read returns UTF32 in this application.
|
|
/// </summary>
|
|
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
|
|
public override int Read()
|
|
{
|
|
int utf16 = Read16();
|
|
if (utf16 < 0xD800 || utf16 > 0xDBFF)
|
|
return utf16;
|
|
else
|
|
{
|
|
int low16 = Read16();
|
|
if (low16 < 0xDC00 || low16 > 0xDFFF)
|
|
return UnicodeReplacementChar;
|
|
else
|
|
return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
|
|
}
|
|
}
|
|
|
|
public override int ReadPos { get { return bPos - 1; } }
|
|
|
|
public override int Peek()
|
|
{
|
|
if (bPos < sLen) return str[bPos];
|
|
else return '\n';
|
|
}
|
|
|
|
public override string GetString(int beg, int end)
|
|
{
|
|
// "end" can be greater than sLen with the BABEL
|
|
// option set. Read returns a "virtual" EOL if
|
|
// an attempt is made to read past the end of the
|
|
// string buffer. Without the guard any attempt
|
|
// to fetch yytext for a token that includes the
|
|
// EOL will throw an index exception.
|
|
if (end > sLen) end = sLen;
|
|
if (end <= beg) return "";
|
|
else return str.Substring(beg, end - beg);
|
|
}
|
|
|
|
public override int Pos
|
|
{
|
|
get { return bPos; }
|
|
set { bPos = value; }
|
|
}
|
|
}
|
|
|
|
#if !NOFILES
|
|
// ====================== Nested class ==========================
|
|
// The LineBuff class contributed by Nigel Horspool,
|
|
// nigelh@cs.uvic.cs
|
|
// ==============================================================
|
|
|
|
public sealed class LineBuff : ScanBuff
|
|
{
|
|
IList<string> line; // list of source lines from a file
|
|
int numLines; // number of strings in line list
|
|
string curLine; // current line in that list
|
|
int cLine; // index of current line in the list
|
|
int curLen; // length of current line
|
|
int curLineStart; // position of line start in whole file
|
|
int curLineEnd; // position of line end in whole file
|
|
int maxPos; // max position ever visited in whole file
|
|
int cPos; // ordinal number of chr in source
|
|
|
|
// Constructed from a list of strings, one per source line.
|
|
// The lines have had trailing '\n' characters removed.
|
|
public LineBuff(IList<string> lineList)
|
|
{
|
|
line = lineList;
|
|
numLines = line.Count;
|
|
cPos = curLineStart = 0;
|
|
curLine = (numLines>0 ? line[0] : "");
|
|
maxPos = curLineEnd = curLen = curLine.Length;
|
|
cLine = 1;
|
|
}
|
|
|
|
public override int Read()
|
|
{
|
|
if (cPos < curLineEnd)
|
|
return curLine[cPos++ - curLineStart];
|
|
if (cPos++ == curLineEnd)
|
|
return '\n';
|
|
if (cLine >= numLines)
|
|
return EOF;
|
|
curLine = line[cLine];
|
|
curLen = curLine.Length;
|
|
curLineStart = curLineEnd + 1;
|
|
curLineEnd = curLineStart + curLen;
|
|
if (curLineEnd>maxPos)
|
|
maxPos = curLineEnd;
|
|
cLine++;
|
|
return curLen>0? curLine[0] : '\n';
|
|
}
|
|
|
|
public override int Peek()
|
|
{
|
|
return (cPos < curLineEnd)? curLine[cPos - curLineStart] : '\n';
|
|
}
|
|
|
|
// To speed up searches for the line containing a position
|
|
private int cachedPos = 0;
|
|
private int cachedIx = 0;
|
|
private int cachedLstart = 0;
|
|
|
|
// Given a position pos within the entire source, the results are
|
|
// ix -- the index of the containing line
|
|
// lstart -- the position of the first character on that line
|
|
private void findIndex( int pos, out int ix, out int lstart )
|
|
{
|
|
if (pos >= cachedPos) {
|
|
ix = cachedIx; lstart = cachedLstart;
|
|
} else {
|
|
ix = lstart = 0;
|
|
}
|
|
for( ; ; ) {
|
|
int len = line[ix].Length + 1;
|
|
if (pos < lstart+len) break;
|
|
lstart += len;
|
|
ix++;
|
|
}
|
|
cachedPos = pos;
|
|
cachedIx = ix;
|
|
cachedLstart = lstart;
|
|
}
|
|
|
|
public override string GetString(int beg, int end)
|
|
{
|
|
if (beg >= maxPos || end <= beg) return "";
|
|
int endIx, begIx, endLineStart, begLineStart;
|
|
findIndex(beg, out begIx, out begLineStart);
|
|
int begCol = beg - begLineStart;
|
|
findIndex(end, out endIx, out endLineStart);
|
|
int endCol = end - endLineStart;
|
|
string s = line[begIx];
|
|
if (begIx == endIx) {
|
|
// the usual case, substring all on one line
|
|
return (endCol <= s.Length)?
|
|
s.Substring(begCol, endCol-begCol)
|
|
: s.Substring(begCol) + "\n";
|
|
}
|
|
// the string spans multiple lines, yuk!
|
|
StringBuilder sb = new StringBuilder();
|
|
if (begCol < s.Length)
|
|
sb.Append(s.Substring(begCol));
|
|
for( ; ; ) {
|
|
sb.Append("\n");
|
|
s = line[++begIx];
|
|
if (begIx >= endIx) break;
|
|
sb.Append(s);
|
|
}
|
|
if (endCol <= s.Length) {
|
|
sb.Append(s.Substring(0, endCol));
|
|
} else {
|
|
sb.Append(s);
|
|
sb.Append("\n");
|
|
}
|
|
return sb.ToString();
|
|
}
|
|
|
|
public override int Pos
|
|
{
|
|
get { return cPos; }
|
|
set {
|
|
cPos = value;
|
|
findIndex(cPos, out cLine, out curLineStart);
|
|
curLine = line[cLine];
|
|
curLineEnd = curLineStart+curLine.Length;
|
|
}
|
|
}
|
|
|
|
public override int ReadPos { get { return cPos - 1; } }
|
|
}
|
|
|
|
// ==============================================================
|
|
// =========== Nested class StreamBuff : byte files =============
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This is the Buffer for byte files. It returns 8-bit characters.
|
|
/// </summary>
|
|
public class StreamBuff : ScanBuff
|
|
{
|
|
protected BufferedStream bStrm; // input buffer
|
|
protected int delta = 1; // number of bytes in chr, could be 0 for EOF.
|
|
|
|
public StreamBuff(Stream str) { this.bStrm = new BufferedStream(str); }
|
|
|
|
/// <summary>
|
|
/// Read returns subset [0 .. 0xFF] in this application.
|
|
/// </summary>
|
|
/// <returns>Unicode point as an int [0 .. 0xFF]; -1 for EOF</returns>
|
|
public override int Read() {
|
|
int ch0 = bStrm.ReadByte();
|
|
delta = (ch0 == EOF ? 0 : 1);
|
|
return ch0;
|
|
}
|
|
|
|
public override int ReadPos {
|
|
get { return (int)bStrm.Position - delta; }
|
|
}
|
|
|
|
public override int Peek()
|
|
{
|
|
int rslt = bStrm.ReadByte();
|
|
bStrm.Seek(-delta, SeekOrigin.Current);
|
|
return rslt;
|
|
}
|
|
|
|
public override string GetString(int beg, int end)
|
|
{
|
|
if (end - beg <= 0) return "";
|
|
long savePos = bStrm.Position;
|
|
char[] arr = new char[end - beg];
|
|
bStrm.Position = (long)beg;
|
|
for (int i = 0; i < (end - beg); i++)
|
|
arr[i] = (char)bStrm.ReadByte();
|
|
bStrm.Position = savePos;
|
|
return new String(arr);
|
|
}
|
|
|
|
// Pos is the position *after* reading chr!
|
|
public override int Pos
|
|
{
|
|
get { return (int)bStrm.Position; }
|
|
set { bStrm.Position = value; }
|
|
}
|
|
|
|
public static int GetCodePage(string command)
|
|
{
|
|
command = command.ToLower();
|
|
if (command.StartsWith("codepage:"))
|
|
command = command.Substring(9);
|
|
try
|
|
{
|
|
if (command.Equals("raw"))
|
|
return -1;
|
|
else if (command.Equals("guess"))
|
|
return -2;
|
|
else if (command.Equals("default"))
|
|
return 0;
|
|
else if (char.IsDigit(command[0]))
|
|
return int.Parse(command);
|
|
else
|
|
{
|
|
Encoding enc = Encoding.GetEncoding(command);
|
|
return enc.CodePage;
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
Console.Error.WriteLine(
|
|
"Unknown codepage \"{0}\", using machine default", command);
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
#if !BYTEMODE
|
|
// ==============================================================
|
|
// ===== Nested class CodePageBuff : for unicode text files =====
|
|
// ========= Byte files mapped by a specified CodePage ==========
|
|
// ==============================================================
|
|
|
|
public sealed class CodePageBuff : StreamBuff
|
|
{
|
|
|
|
char[] map = new char[256];
|
|
|
|
public CodePageBuff(Stream stream) : this(stream, Encoding.Default) { }
|
|
|
|
|
|
public CodePageBuff(Stream stream, Encoding enc) : base(stream)
|
|
{
|
|
bool done;
|
|
int bNum, cNum;
|
|
byte[] bArray = new byte[256];
|
|
if (!enc.IsSingleByte)
|
|
throw new NotImplementedException(
|
|
"Only UTF-8, UTF-16 and single-byte code pages allowed");
|
|
Decoder decoder = enc.GetDecoder();
|
|
for (int i = 0; i < 256; i++)
|
|
bArray[i] = (byte)i;
|
|
decoder.Convert(bArray, 0, 256, this.map, 0, 256, true, out bNum, out cNum, out done);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read returns subset unicode char corresponding to the
|
|
/// given character in the chosen codepage.
|
|
/// </summary>
|
|
/// <returns>Unicode point as an int; -1 for EOF</returns>
|
|
public override int Read() {
|
|
int ch0 = bStrm.ReadByte();
|
|
if (ch0 == EOF)
|
|
{
|
|
delta = 0; return -1;
|
|
}
|
|
else
|
|
{
|
|
delta = 1; return this.map[ch0];
|
|
}
|
|
}
|
|
|
|
public override string GetString(int beg, int end)
|
|
{
|
|
if (end - beg <= 0) return "";
|
|
long savePos = bStrm.Position;
|
|
int saveDelta = delta;
|
|
char[] arr = new char[end - beg];
|
|
bStrm.Position = (long)beg;
|
|
for (int i = 0; i < (end - beg); i++)
|
|
arr[i] = (char)Read();
|
|
bStrm.Position = savePos;
|
|
delta = saveDelta;
|
|
return new String(arr);
|
|
}
|
|
}
|
|
|
|
|
|
// ==============================================================
|
|
// ====== Nested class TextBuff : for unicode text files ========
|
|
// ==== This is the UTF8 class, UTF16 handled by subclasses =====
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This is the Buffer for UTF8 files.
|
|
/// It attempts to read the encoding preamble, which for
|
|
/// this encoding should be unicode point \uFEFF which is
|
|
/// encoded as EF BB BF
|
|
/// </summary>
|
|
public class TextBuff : ScanBuff
|
|
{
|
|
protected BufferedStream bStrm; // input buffer
|
|
protected int delta = 1; // length of chr, zero for EOF!
|
|
|
|
|
|
/// <summary>
|
|
/// TextBuff factory. Reads the file preamble
|
|
/// and returns a TextBuff, LittleEndTextBuff or
|
|
/// BigEndTextBuff according to the result.
|
|
/// </summary>
|
|
/// <param name="strm">The underlying stream</param>
|
|
/// <returns></returns>
|
|
public static ScanBuff NewTextBuff(Stream strm, int fallbackCodepage)
|
|
{
|
|
// First check if this is a UTF16 file
|
|
//
|
|
int b0 = strm.ReadByte();
|
|
int b1 = strm.ReadByte();
|
|
|
|
if (b0 == 0xfe && b1 == 0xff)
|
|
return new BigEndTextBuff(strm);
|
|
if (b0 == 0xff && b1 == 0xfe)
|
|
return new LittleEndTextBuff(strm);
|
|
|
|
int b2 = strm.ReadByte();
|
|
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
|
return new TextBuff(strm);
|
|
//
|
|
// There is no unicode preamble, so
|
|
// we go back to the bytefile default.
|
|
//
|
|
strm.Seek(0, SeekOrigin.Begin);
|
|
|
|
switch (fallbackCodepage)
|
|
{
|
|
case -2:
|
|
{
|
|
int guess = new Guesser.Scanner(strm).GuessCodepage();
|
|
strm.Seek(0, SeekOrigin.Begin);
|
|
if (guess == 0)
|
|
return new CodePageBuff(strm);
|
|
else if (guess == 65001)
|
|
return new TextBuff(strm);
|
|
else
|
|
return new StreamBuff(strm);
|
|
}
|
|
case -1:
|
|
return new StreamBuff(strm);
|
|
case 0:
|
|
return new CodePageBuff(strm); // Default codepage
|
|
case 65001:
|
|
return new TextBuff(strm);
|
|
case 1200:
|
|
return new LittleEndTextBuff(strm);
|
|
case 1201:
|
|
return new BigEndTextBuff(strm);
|
|
default:
|
|
return new CodePageBuff(strm, Encoding.GetEncoding(fallbackCodepage));
|
|
}
|
|
}
|
|
|
|
protected TextBuff(Stream str) {
|
|
this.bStrm = new BufferedStream(str);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read returns UTF32 in this application.
|
|
/// </summary>
|
|
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
|
|
public override int Read()
|
|
{
|
|
int ch0 = bStrm.ReadByte();
|
|
int ch1, ch2, ch3,ch4, ch5;
|
|
int result = UnicodeReplacementChar;
|
|
|
|
if (ch0 < 0x7f)
|
|
{
|
|
delta = (ch0 == EOF ? 0 : 1);
|
|
result = ch0;
|
|
}
|
|
else if ((ch0 & 0xe0) == 0xc0)
|
|
{
|
|
delta = 2;
|
|
ch1 = bStrm.ReadByte();
|
|
if ((ch1 & 0xc0) == 0x80)
|
|
result = ((ch0 & 0x1f) << 6) + (ch1 & 0x3f);
|
|
}
|
|
else if ((ch0 & 0xf0) == 0xe0)
|
|
{
|
|
delta = 3;
|
|
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte();
|
|
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80)
|
|
result = ((ch0 & 0xf) << 12) + ((ch1 & 0x3f) << 6) + (ch2 & 0x3f);
|
|
}
|
|
else if ((ch0 & 0xf8) == 0xf0)
|
|
{
|
|
delta = 4;
|
|
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte(); ch3 = bStrm.ReadByte();
|
|
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && (ch3 & 0xc0) == 0x80)
|
|
{
|
|
result = ((ch0 & 0x7) << 18) + ((ch1 & 0x3f) << 12) +
|
|
((ch2 & 0x3f) << 6) + (ch3 & 0x3f);
|
|
}
|
|
}
|
|
else if ((ch0 & 0xfc) == 0xf8)
|
|
{
|
|
delta = 5;
|
|
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte();
|
|
ch3 = bStrm.ReadByte(); ch4 = bStrm.ReadByte();
|
|
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && (ch3 & 0xc0) == 0x80 && (ch4 & 0xc0) == 0x80)
|
|
{
|
|
result = ((ch0 & 0x3) << 24) + ((ch1 & 0x3f) << 18) +
|
|
((ch2 & 0x3f) << 12) + ((ch3 & 0x3f) << 6) + (ch4 & 0x3f);
|
|
}
|
|
}
|
|
else if ((ch0 & 0xfe) == 0xfc)
|
|
{
|
|
delta = 6;
|
|
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte(); ch3 = bStrm.ReadByte();
|
|
ch4 = bStrm.ReadByte(); ch5 = bStrm.ReadByte();
|
|
|
|
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 &&
|
|
(ch3 & 0xc0) == 0x80 && (ch4 & 0xc0) == 0x80 && (ch5 & 0xc0) == 0x80)
|
|
{
|
|
result = ((ch0 & 0x1) << 28) + ((ch1 & 0x3f) << 24) + ((ch2 & 0x3f) << 18) +
|
|
((ch3 & 0x3f) << 12) + ((ch4 & 0x3f) << 6) + (ch5 & 0x3f);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
public sealed override int ReadPos
|
|
{
|
|
get { return (int)bStrm.Position - delta; }
|
|
}
|
|
|
|
public sealed override int Peek()
|
|
{
|
|
int rslt = Read();
|
|
bStrm.Seek(-delta, SeekOrigin.Current);
|
|
return rslt;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the string from the buffer between
|
|
/// the given file positions. This needs to be
|
|
/// done carefully, as the number of characters
|
|
/// is, in general, not equal to (end - beg).
|
|
/// </summary>
|
|
/// <param name="beg">Begin filepos</param>
|
|
/// <param name="end">End filepos</param>
|
|
/// <returns></returns>
|
|
public override string GetString(int beg, int end)
|
|
{
|
|
int i;
|
|
if (end - beg <= 0) return "";
|
|
long savePos = bStrm.Position;
|
|
int saveDelta = delta;
|
|
//
|
|
// The length of the array cannot be larger than
|
|
// end - beg, since if the characters expand by
|
|
// using surrogate pairs the UTF8 must have taken
|
|
// up at least four bytes for any such character.
|
|
//
|
|
char[] arr = new char[end - beg];
|
|
|
|
bStrm.Position = (long)beg;
|
|
// Normally each call of Read() returns a valid UTF16
|
|
// value which will take up one place in the string.
|
|
// However, if read returns a value > 0xFFFF then a
|
|
// *pair* of surrogate characters must be added.
|
|
for (i = 0; bStrm.Position < end; i++)
|
|
{
|
|
int value = Read();
|
|
if (value < 0xFFFF)
|
|
arr[i] = (char)value;
|
|
else
|
|
{
|
|
int temp = value - 0x10000;
|
|
arr[i++] = (char)(0xD800 + temp / 1024);
|
|
arr[i] = (char)(0xDC00 + temp % 1024);
|
|
}
|
|
}
|
|
bStrm.Position = savePos;
|
|
delta = saveDelta;
|
|
return new String(arr, 0, i);
|
|
}
|
|
|
|
// Pos is the position *after* reading chr!
|
|
public sealed override int Pos
|
|
{
|
|
get { return (int)bStrm.Position; }
|
|
set { bStrm.Position = value; }
|
|
}
|
|
}
|
|
|
|
// ==============================================================
|
|
// ====================== Nested class ==========================
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This is the Buffer for Big-endian UTF16 files.
|
|
/// </summary>
|
|
public sealed class BigEndTextBuff : TextBuff
|
|
{
|
|
internal BigEndTextBuff(Stream str) : base(str) { } //
|
|
|
|
private int Read16()
|
|
{
|
|
int ch0 = bStrm.ReadByte();
|
|
int ch1 = bStrm.ReadByte();
|
|
if (ch1 == EOF)
|
|
{
|
|
// An EOF in either byte counts as an EOF
|
|
delta = (ch0 == EOF ? 0 : 1);
|
|
return -1;
|
|
}
|
|
else
|
|
{
|
|
delta = 2;
|
|
return (ch0 << 8) + ch1;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read returns UTF32 in this application.
|
|
/// </summary>
|
|
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
|
|
public override int Read()
|
|
{
|
|
int utf16 = Read16();
|
|
if (utf16 < 0xD800 || utf16 > 0xDBFF)
|
|
return utf16;
|
|
else
|
|
{
|
|
int low16 = Read16();
|
|
delta = 4;
|
|
if (low16 < 0xDC00 || low16 > 0xDFFF)
|
|
return UnicodeReplacementChar;
|
|
else
|
|
return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the string from the buffer between
|
|
/// the given file positions. This needs to be
|
|
/// done carefully, as the number of characters
|
|
/// is, in general, not equal to (end - beg).
|
|
/// </summary>
|
|
/// <param name="beg">Begin filepos</param>
|
|
/// <param name="end">End filepos</param>
|
|
/// <returns></returns>
|
|
public sealed override string GetString(int beg, int end)
|
|
{
|
|
int i;
|
|
if (end - beg <= 0) return "";
|
|
long savePos = bStrm.Position;
|
|
int saveDelta = delta;
|
|
char[] arr = new char[end - beg];
|
|
bStrm.Position = (long)beg;
|
|
// This stream will have utf16 valid for strings
|
|
for (i = 0; bStrm.Position < end; i++)
|
|
arr[i] = (char)Read();
|
|
bStrm.Position = savePos;
|
|
delta = saveDelta;
|
|
return new String(arr, 0, i);
|
|
}
|
|
}
|
|
|
|
// ==============================================================
|
|
// ====================== Nested class ==========================
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// This is the Buffer for Little-endian UTF16 files.
|
|
/// </summary>
|
|
public sealed class LittleEndTextBuff : TextBuff
|
|
{
|
|
internal LittleEndTextBuff(Stream str) : base(str) { } // { this.bStrm = new BufferedStream(str); }
|
|
|
|
private int Read16()
|
|
{
|
|
int ch0 = bStrm.ReadByte();
|
|
int ch1 = bStrm.ReadByte();
|
|
if (ch1 == EOF)
|
|
{
|
|
// An EOF in either byte counts as an EOF
|
|
delta = (ch0 == EOF ? 0 : 1);
|
|
return -1;
|
|
}
|
|
else
|
|
{
|
|
delta = 2;
|
|
return (ch1 << 8) + ch0;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read returns UTF32 in this application.
|
|
/// </summary>
|
|
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
|
|
public override int Read()
|
|
{
|
|
int utf16 = Read16();
|
|
if (utf16 < 0xD800 || utf16 > 0xDBFF)
|
|
return utf16;
|
|
else
|
|
{
|
|
int low16 = Read16();
|
|
delta = 4;
|
|
if (low16 < 0xDC00 || low16 > 0xDFFF)
|
|
return UnicodeReplacementChar;
|
|
else
|
|
return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the string from the buffer between
|
|
/// the given file positions. This needs to be
|
|
/// done carefully, as the number of characters
|
|
/// is, in general, not equal to (end - beg).
|
|
/// </summary>
|
|
/// <param name="beg">Begin filepos</param>
|
|
/// <param name="end">End filepos</param>
|
|
/// <returns></returns>
|
|
public sealed override string GetString(int beg, int end)
|
|
{
|
|
int i;
|
|
if (end - beg <= 0) return "";
|
|
long savePos = bStrm.Position;
|
|
int saveDelta = delta;
|
|
char[] arr = new char[end - beg];
|
|
bStrm.Position = (long)beg;
|
|
// This stream will have utf16 valid for strings
|
|
for (i = 0; bStrm.Position < end; i++)
|
|
arr[i] = (char)Read16();
|
|
bStrm.Position = savePos;
|
|
delta = saveDelta;
|
|
return new String(arr, 0, i);
|
|
}
|
|
}
|
|
#endif // !BYTEMODE
|
|
#endif // !NOFILES
|
|
#endregion Buffer classes
|
|
|
|
// =================== End Nested classes =======================
|
|
|
|
#if !NOFILES
|
|
public Scanner(Stream file) {
|
|
SetSource(file); // no unicode option
|
|
}
|
|
|
|
public Scanner(Stream file, string codepage) {
|
|
SetSource(file); // Not unicode, ignoring codepage
|
|
}
|
|
|
|
#endif // !NOFILES
|
|
|
|
public Scanner() { }
|
|
|
|
void GetChr()
|
|
{
|
|
if (chr == '\n') // This needs to be fixed for other conventions
|
|
{
|
|
lineStartNum = cNum + 1;
|
|
lNum++;
|
|
}
|
|
chr = buffer.Read();
|
|
cNum++;
|
|
}
|
|
|
|
void MarkToken()
|
|
{
|
|
tokPos = buffer.ReadPos;
|
|
tokNum = cNum;
|
|
tokLin = lNum;
|
|
tokCol = cNum - lineStartNum;
|
|
}
|
|
|
|
void MarkEnd()
|
|
{
|
|
tokTxt = null;
|
|
tokLen = cNum - tokNum;
|
|
tokEPos = buffer.ReadPos;
|
|
tokELin = lNum;
|
|
tokECol = cNum - lineStartNum;
|
|
}
|
|
|
|
// ==============================================================
|
|
// ===== Initialization of string-based input buffers ====
|
|
// ==============================================================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a StringBuff buffer object for this scanner
|
|
/// </summary>
|
|
/// <param name="source">the input string</param>
|
|
/// <param name="offset">starting offset in the string</param>
|
|
public void SetSource(string source, int offset)
|
|
{
|
|
this.buffer = new StringBuff(source);
|
|
this.buffer.Pos = offset;
|
|
this.lNum = 0;
|
|
this.cNum = offset - 1;
|
|
this.chr = '\n'; // to initialize yyline, yycol and lineStart
|
|
GetChr();
|
|
}
|
|
|
|
#if !NOFILES
|
|
// ================ LineBuffer Initialization ===================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a LineBuff buffer object for this scanner
|
|
/// </summary>
|
|
/// <param name="source">the list of input strings</param>
|
|
public void SetSource(IList<string> source)
|
|
{
|
|
this.buffer = new LineBuff(source);
|
|
this.chr = '\n'; // to initialize yyline, yycol and lineStart
|
|
this.lNum = 0;
|
|
this.cNum = -1;
|
|
GetChr();
|
|
}
|
|
|
|
// =============== StreamBuffer Initialization ==================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a StreamBuff buffer object for this scanner.
|
|
/// StreamBuff is buffer for 8-bit byte files.
|
|
/// </summary>
|
|
/// <param name="source">the input byte stream</param>
|
|
public void SetSource(Stream source)
|
|
{
|
|
this.buffer = new StreamBuff(source);
|
|
this.lNum = 0;
|
|
this.cNum = -1;
|
|
this.chr = '\n'; // to initialize yyline, yycol and lineStart
|
|
GetChr();
|
|
}
|
|
|
|
#if !BYTEMODE
|
|
// ================ TextBuffer Initialization ===================
|
|
|
|
/// <summary>
|
|
/// Create and initialize a TextBuff buffer object for this scanner.
|
|
/// TextBuff is a buffer for encoded unicode files.
|
|
/// </summary>
|
|
/// <param name="source">the input text file</param>
|
|
/// <param name="fallbackCodepage">Codepage to use if file has
|
|
/// no BOM. For 0, use machine default; for -1, 8-bit binary</param>
|
|
public void SetSource(Stream source, int fallbackCodepage)
|
|
{
|
|
this.buffer = TextBuff.NewTextBuff(source, fallbackCodepage);
|
|
this.lNum = 0;
|
|
this.cNum = -1;
|
|
this.chr = '\n'; // to initialize yyline, yycol and lineStart
|
|
GetChr();
|
|
}
|
|
#endif // !BYTEMODE
|
|
#endif // !NOFILES
|
|
|
|
// ==============================================================
|
|
|
|
#if BABEL
|
|
//
|
|
// Get the next token for Visual Studio
|
|
//
|
|
// "state" is the inout mode variable that maintains scanner
|
|
// state between calls, using the EolState property. In principle,
|
|
// if the calls of EolState are costly set could be called once
|
|
// only per line, at the start; and get called only at the end
|
|
// of the line. This needs more infrastructure ...
|
|
//
|
|
public int GetNext(ref int state, out int start, out int end)
|
|
{
|
|
Tokens next;
|
|
int s, e;
|
|
s = state; // state at start
|
|
EolState = state;
|
|
next = (Tokens)Scan();
|
|
state = EolState;
|
|
e = state; // state at end;
|
|
start = tokPos;
|
|
end = tokEPos - 1; // end is the index of last char.
|
|
return (int)next;
|
|
}
|
|
#endif // BABEL
|
|
|
|
// ======== IScanner<> Implementation =========
|
|
|
|
public override int yylex()
|
|
{
|
|
// parserMax is set by reflecting on the Tokens
|
|
// enumeration. If maxParseToken is defined
|
|
// that is used, otherwise int.MaxValue is used.
|
|
int next;
|
|
do { next = Scan(); } while (next >= parserMax);
|
|
return next;
|
|
}
|
|
|
|
int yyleng { get { return tokLen; } }
|
|
int yypos { get { return tokPos; } }
|
|
int yyline { get { return tokLin; } }
|
|
int yycol { get { return tokCol; } }
|
|
|
|
public string yytext
|
|
{
|
|
get
|
|
{
|
|
if (tokTxt == null)
|
|
tokTxt = buffer.GetString(tokPos, tokEPos);
|
|
return tokTxt;
|
|
}
|
|
}
|
|
|
|
void yyless(int n) {
|
|
buffer.Pos = tokPos;
|
|
// Must read at least one char, so set before start.
|
|
cNum = tokNum - 1; GetChr();
|
|
// Now ensure that line counting is correct.
|
|
lNum = tokLin;
|
|
// And count the rest of the text.
|
|
for (int i = 0; i < n; i++) GetChr();
|
|
MarkEnd();
|
|
}
|
|
|
|
// ============ methods available in actions ==============
|
|
|
|
internal int YY_START {
|
|
get { return currentScOrd; }
|
|
set { currentScOrd = value;
|
|
currentStart = startState[value];
|
|
}
|
|
}
|
|
|
|
internal void BEGIN(int next) {
|
|
currentScOrd = next;
|
|
currentStart = startState[next];
|
|
}
|
|
|
|
// ============== The main tokenizer code =================
|
|
|
|
int Scan()
|
|
{
|
|
try {
|
|
for (; ; )
|
|
{
|
|
int next; // next state to enter
|
|
#if BACKUP
|
|
Result rslt = Result.noMatch;
|
|
#endif // BACKUP
|
|
#if LEFTANCHORS
|
|
for (;;)
|
|
{
|
|
// Discard characters that do not start any pattern.
|
|
// Must check the left anchor condition after *every* GetChr!
|
|
state = (lineStartNum == cNum ? anchorState[currentScOrd] : currentStart);
|
|
if ((next = NextState()) != goStart)
|
|
break; // LOOP EXIT HERE...
|
|
GetChr();
|
|
}
|
|
|
|
#else // !LEFTANCHORS
|
|
state = currentStart;
|
|
while ((next = NextState()) == goStart)
|
|
// At this point, the current character has no
|
|
// transition from the current state. We discard
|
|
// the "no-match" char. In traditional LEX such
|
|
// characters are echoed to the console.
|
|
GetChr();
|
|
#endif // LEFTANCHORS
|
|
// At last, a valid transition ...
|
|
MarkToken();
|
|
state = next;
|
|
GetChr();
|
|
|
|
while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum
|
|
#if BACKUP
|
|
if (state <= maxAccept && next > maxAccept) // need to prepare backup data
|
|
{
|
|
// ctx is an object. The fields may be
|
|
// mutated by the call to Recurse2.
|
|
// On return the data in ctx is the
|
|
// *latest* accept state that was found.
|
|
Context ctx = new Context();
|
|
rslt = Recurse2(ctx, next);
|
|
if (rslt == Result.noMatch)
|
|
RestoreStateAndPos(ctx);
|
|
break;
|
|
}
|
|
else
|
|
#endif // BACKUP
|
|
{
|
|
state = next;
|
|
GetChr();
|
|
}
|
|
if (state <= maxAccept)
|
|
{
|
|
MarkEnd();
|
|
#region ActionSwitch
|
|
#pragma warning disable 162
|
|
switch (state)
|
|
{
|
|
case eofNum:
|
|
if (yywrap())
|
|
return (int)Tokens.EOF;
|
|
break;
|
|
case 1:
|
|
;
|
|
break;
|
|
case 2:
|
|
case 3:
|
|
return (int)Tokens.ENDLINE;
|
|
break;
|
|
case 4:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.LPAREN;
|
|
break;
|
|
case 5:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.RPAREN;
|
|
break;
|
|
case 6:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Multiplication, yylloc);
|
|
|
|
return (int)Tokens.MULT;
|
|
break;
|
|
case 7:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Plus, yylloc);
|
|
|
|
return (int)Tokens.PLUS;
|
|
break;
|
|
case 8:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.COLUMN;
|
|
break;
|
|
case 9:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Minus, yylloc);
|
|
|
|
return (int)Tokens.MINUS;
|
|
break;
|
|
case 10:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.DOT;
|
|
break;
|
|
case 11:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Division, yylloc);
|
|
|
|
return (int)Tokens.DIVIDE;
|
|
break;
|
|
case 12:
|
|
yylval._const_node = PT.create_int_const(yytext, yylloc);
|
|
|
|
return (int)Tokens.INTNUM;
|
|
break;
|
|
case 13:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.COLON;
|
|
break;
|
|
case 14:
|
|
return (int)Tokens.SEMICOLUMN;
|
|
break;
|
|
case 15:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Less, yylloc);
|
|
|
|
return (int)Tokens.LT;
|
|
break;
|
|
case 16:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Assignment, yylloc);
|
|
|
|
return (int)Tokens.ASSIGN;
|
|
break;
|
|
case 17:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Greater, yylloc);
|
|
|
|
return (int)Tokens.GT;
|
|
break;
|
|
case 18:
|
|
case 19:
|
|
case 20:
|
|
case 23:
|
|
case 24:
|
|
case 25:
|
|
case 27:
|
|
case 29:
|
|
case 31:
|
|
case 32:
|
|
case 34:
|
|
case 35:
|
|
case 36:
|
|
if (ScannerHelper.keywords.ContainsKey(yytext))
|
|
{
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return ScannerHelper.keywords[yytext];
|
|
}
|
|
else
|
|
{
|
|
yylval._ident = PT.create_directive_name(yytext, yylloc);
|
|
|
|
return (int)Tokens.ID;
|
|
}
|
|
break;
|
|
case 21:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.LSQUARE;
|
|
break;
|
|
case 22:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.RSQUARE;
|
|
break;
|
|
case 26:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.LogicalOR, yylloc);
|
|
|
|
return (int)Tokens.OR;
|
|
break;
|
|
case 28:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.LogicalNOT, yylloc);
|
|
|
|
return (int)Tokens.NOT;
|
|
break;
|
|
case 30:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.LogicalAND, yylloc);
|
|
|
|
return (int)Tokens.AND;
|
|
break;
|
|
case 33:
|
|
yylval._const_node = PT.create_bool_const(true, yylloc);
|
|
|
|
return (int)Tokens.TRUECONST;
|
|
break;
|
|
case 37:
|
|
yylval._const_node = PT.create_bool_const(false, yylloc);
|
|
|
|
return (int)Tokens.FALSECONST;
|
|
break;
|
|
case 38:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.GreaterEqual, yylloc);
|
|
|
|
return (int)Tokens.GE;
|
|
break;
|
|
case 39:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.Equal, yylloc);
|
|
|
|
return (int)Tokens.EQ;
|
|
break;
|
|
case 40:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.LessEqual, yylloc);
|
|
|
|
return (int)Tokens.LE;
|
|
break;
|
|
case 41:
|
|
yylval._const_node = PT.create_double_const(yytext, yylloc);
|
|
|
|
return (int)Tokens.FLOATNUM;
|
|
break;
|
|
case 42:
|
|
yylval._token_info = PT.create_token_info(yytext, yylloc);
|
|
|
|
return (int)Tokens.FUNCRETSYMB;
|
|
break;
|
|
case 43:
|
|
case 45:
|
|
case 48:
|
|
yylval._const_node = PT.create_string_const(yytext, yylloc);
|
|
|
|
return (int)Tokens.STRINGLITERAL;
|
|
break;
|
|
case 44:
|
|
yylval._const_node = PT.create_string_const(yytext, yylloc);
|
|
|
|
return (int)Tokens.MULTILINESTRING;
|
|
break;
|
|
case 46:
|
|
case 47:
|
|
BEGIN(INITIAL);
|
|
break;
|
|
case 49:
|
|
yylval._op_type_node = PT.create_op_type_node(Operators.NotEqual, yylloc);
|
|
|
|
return (int)Tokens.NE;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
#pragma warning restore 162
|
|
#endregion
|
|
}
|
|
}
|
|
} // end try
|
|
finally {
|
|
// User-specified epilog to scan()
|
|
yylloc = new LexLocation(tokLin, tokCol, tokELin, tokECol);
|
|
// End, user-specified epilog
|
|
} // end finally
|
|
}
|
|
|
|
#if BACKUP
|
|
Result Recurse2(Context ctx, int next)
|
|
{
|
|
// Assert: at entry "state" is an accept state AND
|
|
// NextState(state, chr) != goStart AND
|
|
// NextState(state, chr) is not an accept state.
|
|
//
|
|
SaveStateAndPos(ctx);
|
|
state = next;
|
|
// if (state == eofNum) return Result.accept;
|
|
GetChr();
|
|
|
|
while ((next = NextState()) > eofNum)
|
|
{
|
|
if (state <= maxAccept && next > maxAccept) // need to update backup data
|
|
SaveStateAndPos(ctx);
|
|
state = next;
|
|
if (state == eofNum) return Result.accept;
|
|
GetChr();
|
|
}
|
|
return (state <= maxAccept ? Result.accept : Result.noMatch);
|
|
}
|
|
|
|
void SaveStateAndPos(Context ctx)
|
|
{
|
|
ctx.bPos = buffer.Pos;
|
|
ctx.cNum = cNum;
|
|
ctx.lNum = lNum;
|
|
ctx.state = state;
|
|
ctx.cChr = chr;
|
|
}
|
|
|
|
void RestoreStateAndPos(Context ctx)
|
|
{
|
|
buffer.Pos = ctx.bPos;
|
|
cNum = ctx.cNum;
|
|
lNum = ctx.lNum;
|
|
state = ctx.state;
|
|
chr = ctx.cChr;
|
|
}
|
|
|
|
void RestorePos(Context ctx) { buffer.Pos = ctx.bPos; cNum = ctx.cNum; }
|
|
#endif // BACKUP
|
|
|
|
// ============= End of the tokenizer code ================
|
|
|
|
#if STACK
|
|
internal void yy_clear_stack() { scStack.Clear(); }
|
|
internal int yy_top_state() { return scStack.Peek(); }
|
|
|
|
internal void yy_push_state(int state)
|
|
{
|
|
scStack.Push(currentScOrd);
|
|
BEGIN(state);
|
|
}
|
|
|
|
internal void yy_pop_state()
|
|
{
|
|
// Protect against input errors that pop too far ...
|
|
if (scStack.Count > 0) {
|
|
int newSc = scStack.Pop();
|
|
BEGIN(newSc);
|
|
} // Otherwise leave stack unchanged.
|
|
}
|
|
#endif // STACK
|
|
|
|
internal void ECHO() { Console.Out.Write(yytext); }
|
|
|
|
#region UserCodeSection
|
|
|
|
class ScannerHelper
|
|
{
|
|
public static Dictionary<string, int> keywords;
|
|
|
|
static ScannerHelper()
|
|
{
|
|
keywords = new Dictionary<string, int>();
|
|
|
|
keywords.Add("python",(int)Tokens.kINDENT);
|
|
keywords.Add("nohtyp",(int)Tokens.kDEDENT);
|
|
|
|
keywords.Add("if",(int)Tokens.kIF);
|
|
keywords.Add("else",(int)Tokens.kELSE);
|
|
keywords.Add("while",(int)Tokens.kWHILE);
|
|
|
|
keywords.Add("and",(int)Tokens.AND);
|
|
keywords.Add("or",(int)Tokens.OR);
|
|
keywords.Add("not",(int)Tokens.NOT);
|
|
|
|
keywords.Add("def",(int)Tokens.kDEF);
|
|
keywords.Add("fed",(int)Tokens.kFED);
|
|
|
|
keywords.Add("return",(int)Tokens.kRETURN);
|
|
|
|
keywords.Add("int", (int)Tokens.bINT);
|
|
keywords.Add("float", (int)Tokens.bFLOAT);
|
|
keywords.Add("str", (int)Tokens.bSTR);
|
|
keywords.Add("bool", (int)Tokens.bBOOL);
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
} // end class Scanner
|
|
} // end namespace
|