administrator/pascalabcnet
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity
pascalabcnet/Languages/OldPython/PythonParser/python_gppg_lex.cs

1717 lines
63 KiB
C#
Raw Normal View History

initial commit
2015-05-14 22:35:07 +03:00
//
// This CSharp output file generated by Gardens Point LEX
// Version: 1.0.1.250 (2009-01-01)
// Machine: PC
// DateTime: 30.06.2009 22:50:21
// UserName: A
// GPLEX input file <python_lex.lex>
// GPLEX frame file <gplexx.frame>
//
// Option settings: parser, minimize, compressnext
//
//
// gplexx.frame
// Version 1.0.0 of 01-November-2008
//
#define BACKUP
#define BYTEMODE
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
#if !STANDALONE
using gppg;
#endif
using PascalABCCompiler.SyntaxTree;
using System.Globalization;
using GPPGTools;
namespace PascalABCCompiler.PythonABCParser
{
/// <summary>
/// Summary Canonical example of GPLEX automaton
/// </summary>
#if STANDALONE
//
// These are the dummy declarations for stand-alone GPLEX applications
// normally these declarations would come from the parser.
// If you declare /noparser, or %option noparser then you get this.
//
public enum Tokens
{
EOF = 0, maxParseToken = int.MaxValue
// must have at least these two, values are almost arbitrary
}
public abstract class ScanBase
{
public abstract int yylex();
protected virtual bool yywrap() { return true; }
#if BABEL
protected abstract int CurrentSc { get; set; }
// EolState is the 32-bit of state data persisted at
// the end of each line for Visual Studio colorization.
// The default is to return CurrentSc. You must override
// this if you want more complicated behavior.
public virtual int EolState {
get { return CurrentSc; }
set { CurrentSc = value; }
}
}
public interface IColorScan
{
void SetSource(string source, int offset);
int GetNext(ref int state, out int start, out int end);
#endif // BABEL
}
#endif // STANDALONE
public abstract class ScanBuff
{
public const int EOF = -1;
public abstract int Pos { get; set; }
public abstract int Read();
public abstract int Peek();
public abstract int ReadPos { get; }
public abstract string GetString(int b, int e);
public const int UnicodeReplacementChar = 0xFFFD;
}
// If the compiler can't find ScanBase maybe you need to run
// GPPG with the /gplex option, or GPLEX with /noparser
#if BABEL
public sealed partial class Scanner : ScanBase, IColorScan
{
public ScanBuff buffer;
int currentScOrd = 0; // start condition ordinal
protected override int CurrentSc
{
// The current start state is a property
// to try to avoid the user error of setting
// scState but forgetting to update the FSA
// start state "currentStart"
//
get { return currentScOrd; } // i.e. return YY_START;
set { currentScOrd = value; // i.e. BEGIN(value);
currentStart = startState[value]; }
}
#else // BABEL
public sealed partial class Scanner : ScanBase
{
public ScanBuff buffer;
int currentScOrd = 0; // start condition ordinal
#endif // BABEL
private static int GetMaxParseToken() {
System.Reflection.FieldInfo f = typeof(Tokens).GetField("maxParseToken");
return (f == null ? int.MaxValue : (int)f.GetValue(null));
}
static int parserMax = GetMaxParseToken();
enum Result {accept, noMatch, contextFound};
const int maxAccept = 49;
const int initial = 50;
const int eofNum = 0;
const int goStart = -1;
const int INITIAL = 0;
#region user code
PythonParserTools PT = new PythonParserTools();
#endregion user code
int state;
int currentStart = startState[0];
int chr; // last character read
int cNum = 0; // ordinal number of chr
int lNum = 0; // current line number
int lineStartNum; // ordinal number at start of line
//
// The following instance variables are used, among other
// things, for constructing the yylloc location objects.
//
int tokPos; // buffer position at start of token
int tokNum; // ordinal number of first character
int tokLen; // number of characters in token
int tokCol; // zero-based column number at start of token
int tokLin; // line number at start of token
int tokEPos; // buffer position at end of token
int tokECol; // column number at end of token
int tokELin; // line number at end of token
string tokTxt; // lazily constructed text of token
#if STACK
private Stack<int> scStack = new Stack<int>();
#endif // STACK
#region ScannerTables
struct Table {
public int min; public int rng; public int dflt;
public sbyte[] nxt;
public Table(int m, int x, int d, sbyte[] n) {
min = m; rng = x; dflt = d; nxt = n;
}
};
static int[] startState = {50, 0};
static Table[] NxS = new Table[64];
static Scanner() {
NxS[0] = new Table(0, 0, 0, null);
NxS[1] = new Table(0, 0, -1, null);
NxS[2] = new Table(0, 0, -1, null);
NxS[3] = new Table(10, 1, -1, new sbyte[] {2});
NxS[4] = new Table(0, 0, -1, null);
NxS[5] = new Table(0, 0, -1, null);
NxS[6] = new Table(0, 0, -1, null);
NxS[7] = new Table(0, 0, -1, null);
NxS[8] = new Table(0, 0, -1, null);
NxS[9] = new Table(62, 1, -1, new sbyte[] {42});
NxS[10] = new Table(0, 0, -1, null);
NxS[11] = new Table(0, 0, -1, null);
NxS[12] = new Table(46, 12, -1, new sbyte[] {55, -1, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12});
NxS[13] = new Table(0, 0, -1, null);
NxS[14] = new Table(0, 0, -1, null);
NxS[15] = new Table(61, 1, -1, new sbyte[] {40});
NxS[16] = new Table(61, 1, -1, new sbyte[] {39});
NxS[17] = new Table(61, 1, -1, new sbyte[] {38});
NxS[18] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[19] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 34, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[20] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 31, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[21] = new Table(0, 0, -1, null);
NxS[22] = new Table(0, 0, -1, null);
NxS[23] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 29, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[24] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 27, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[25] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 26, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[26] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[27] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 28, 18,
18, 18, 18, 18, 18});
NxS[28] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[29] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 30, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[30] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[31] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 32,
18, 18, 18, 18, 18});
NxS[32] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 33,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[33] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[34] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 35, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[35] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 36, 18, 18,
18, 18, 18, 18, 18});
NxS[36] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 37,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[37] = new Table(48, 75, -1, new sbyte[] {18, 18, 18, 18, 18, 18,
18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, -1, -1, -1, -1, 18, -1, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18});
NxS[38] = new Table(0, 0, -1, null);
NxS[39] = new Table(0, 0, -1, null);
NxS[40] = new Table(0, 0, -1, null);
NxS[41] = new Table(48, 10, -1, new sbyte[] {41, 41, 41, 41, 41, 41,
41, 41, 41, 41});
NxS[42] = new Table(0, 0, -1, null);
NxS[43] = new Table(39, 1, -1, new sbyte[] {57});
NxS[44] = new Table(0, 0, -1, null);
NxS[45] = new Table(0, 0, -1, null);
NxS[46] = new Table(0, 0, -1, null);
NxS[47] = new Table(10, 4, 53, new sbyte[] {46, 53, 53, 47});
NxS[48] = new Table(34, 1, -1, new sbyte[] {61});
NxS[49] = new Table(0, 0, -1, null);
NxS[50] = new Table(9, 114, -1, new sbyte[] {1, 2, -1, -1, 3, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, 1, 51, 52, 53, -1, -1, -1, 54, 4, 5, 6, 7, 8, 9, 10,
11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 16, 17,
-1, -1, 18, 18, 18, 18, 18, 19, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 20, 18, 18, 18, 18, 18, 18, 21, -1, 22, -1,
18, -1, 23, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 24,
25, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18});
NxS[51] = new Table(61, 1, -1, new sbyte[] {49});
NxS[52] = new Table(10, 25, 60, new sbyte[] {-1, 60, 60, 60, 60, 60,
60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60,
60, 60, 48});
NxS[53] = new Table(10, 4, 53, new sbyte[] {46, 53, 53, 47});
NxS[54] = new Table(10, 30, 56, new sbyte[] {-1, 56, 56, 56, 56, 56,
56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
56, 56, 56, 56, 56, 56, 56, 43});
NxS[55] = new Table(48, 10, -1, new sbyte[] {41, 41, 41, 41, 41, 41,
41, 41, 41, 41});
NxS[56] = new Table(10, 30, 56, new sbyte[] {-1, 56, 56, 56, 56, 56,
56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
56, 56, 56, 56, 56, 56, 56, 45});
NxS[57] = new Table(39, 1, 57, new sbyte[] {58});
NxS[58] = new Table(39, 1, -1, new sbyte[] {59});
NxS[59] = new Table(39, 1, -1, new sbyte[] {44});
NxS[60] = new Table(10, 25, 60, new sbyte[] {-1, 60, 60, 60, 60, 60,
60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60,
60, 60, 45});
NxS[61] = new Table(34, 1, 61, new sbyte[] {62});
NxS[62] = new Table(34, 1, -1, new sbyte[] {63});
NxS[63] = new Table(34, 1, -1, new sbyte[] {44});
}
int NextState(int qStat) {
if (chr == ScanBuff.EOF)
return eofNum;
else {
int rslt;
int idx = (byte)(chr - NxS[qStat].min);
if ((uint)idx >= (uint)NxS[qStat].rng) rslt = NxS[qStat].dflt;
else rslt = NxS[qStat].nxt[idx];
return rslt;
}
}
int NextState() {
if (chr == ScanBuff.EOF)
return eofNum;
else {
int rslt;
int idx = (byte)(chr - NxS[state].min);
if ((uint)idx >= (uint)NxS[state].rng) rslt = NxS[state].dflt;
else rslt = NxS[state].nxt[idx];
return rslt;
}
}
// EXPERIMENTAL: This is the NextState method that
// is used by the CharClassPredicate functions for V0.9
int TestNextState(int qStat, int chr) {
if (chr == ScanBuff.EOF)
return eofNum;
else {
int rslt;
int idx = (byte)(chr - NxS[qStat].min);
if ((uint)idx >= (uint)NxS[qStat].rng) rslt = NxS[qStat].dflt;
else rslt = NxS[qStat].nxt[idx];
return rslt;
}
}
#endregion
#if BACKUP
// ==============================================================
// == Nested class used for backup in automata which do backup ==
// ==============================================================
internal class Context // class used for automaton backup.
{
public int bPos;
public int cNum;
public int lNum; // Need this in case of backup over EOL.
public int state;
public int cChr;
}
#endif // BACKUP
// ==============================================================
// ==== Nested struct to support input switching in scanners ====
// ==============================================================
struct BufferContext {
internal ScanBuff buffSv;
internal int chrSv;
internal int cNumSv;
internal int lNumSv;
internal int startSv;
}
#region Buffer classes
// ==============================================================
// ===== Private methods to save and restore buffer contexts ====
// ==============================================================
/// <summary>
/// This method creates a buffer context record from
/// the current buffer object, together with some
/// scanner state values.
/// </summary>
BufferContext MkBuffCtx()
{
BufferContext rslt;
rslt.buffSv = this.buffer;
rslt.chrSv = this.chr;
rslt.cNumSv = this.cNum;
rslt.lNumSv = this.lNum;
rslt.startSv = this.lineStartNum;
return rslt;
}
/// <summary>
/// This method restores the buffer value and allied
/// scanner state from the given context record value.
/// </summary>
void RestoreBuffCtx(BufferContext value)
{
this.buffer = value.buffSv;
this.chr = value.chrSv;
this.cNum = value.cNumSv;
this.lNum = value.lNumSv;
this.lineStartNum = value.startSv;
}
// ==============================================================
// ===== Nested classes for various ScanBuff derived classes ====
// ==============================================================
// =============== Nested class for string input ================
// ==============================================================
/// <summary>
/// This class read characters from a single string as
/// required, for example, by Visual Studio language services
/// </summary>
public sealed class StringBuff : ScanBuff
{
string str; // input buffer
int bPos; // current position in buffer
int sLen;
public StringBuff(string str)
{
this.str = str;
this.sLen = str.Length;
}
private int Read16()
{
if (bPos < sLen) return str[bPos++];
#if BABEL
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see newline
#endif // BABEL
else { bPos++; return EOF; } // two strikes and you're out!
}
/// <summary>
/// Read returns UTF32 in this application.
/// </summary>
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
public override int Read()
{
int utf16 = Read16();
if (utf16 < 0xD800 || utf16 > 0xDBFF)
return utf16;
else
{
int low16 = Read16();
if (low16 < 0xDC00 || low16 > 0xDFFF)
return UnicodeReplacementChar;
else
return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
}
}
public override int ReadPos { get { return bPos - 1; } }
public override int Peek()
{
if (bPos < sLen) return str[bPos];
else return '\n';
}
public override string GetString(int beg, int end)
{
// "end" can be greater than sLen with the BABEL
// option set. Read returns a "virtual" EOL if
// an attempt is made to read past the end of the
// string buffer. Without the guard any attempt
// to fetch yytext for a token that includes the
// EOL will throw an index exception.
if (end > sLen) end = sLen;
if (end <= beg) return "";
else return str.Substring(beg, end - beg);
}
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
}
#if !NOFILES
// ====================== Nested class ==========================
// The LineBuff class contributed by Nigel Horspool,
// nigelh@cs.uvic.cs
// ==============================================================
public sealed class LineBuff : ScanBuff
{
IList<string> line; // list of source lines from a file
int numLines; // number of strings in line list
string curLine; // current line in that list
int cLine; // index of current line in the list
int curLen; // length of current line
int curLineStart; // position of line start in whole file
int curLineEnd; // position of line end in whole file
int maxPos; // max position ever visited in whole file
int cPos; // ordinal number of chr in source
// Constructed from a list of strings, one per source line.
// The lines have had trailing '\n' characters removed.
public LineBuff(IList<string> lineList)
{
line = lineList;
numLines = line.Count;
cPos = curLineStart = 0;
curLine = (numLines>0 ? line[0] : "");
maxPos = curLineEnd = curLen = curLine.Length;
cLine = 1;
}
public override int Read()
{
if (cPos < curLineEnd)
return curLine[cPos++ - curLineStart];
if (cPos++ == curLineEnd)
return '\n';
if (cLine >= numLines)
return EOF;
curLine = line[cLine];
curLen = curLine.Length;
curLineStart = curLineEnd + 1;
curLineEnd = curLineStart + curLen;
if (curLineEnd>maxPos)
maxPos = curLineEnd;
cLine++;
return curLen>0? curLine[0] : '\n';
}
public override int Peek()
{
return (cPos < curLineEnd)? curLine[cPos - curLineStart] : '\n';
}
// To speed up searches for the line containing a position
private int cachedPos = 0;
private int cachedIx = 0;
private int cachedLstart = 0;
// Given a position pos within the entire source, the results are
// ix -- the index of the containing line
// lstart -- the position of the first character on that line
private void findIndex( int pos, out int ix, out int lstart )
{
if (pos >= cachedPos) {
ix = cachedIx; lstart = cachedLstart;
} else {
ix = lstart = 0;
}
for( ; ; ) {
int len = line[ix].Length + 1;
if (pos < lstart+len) break;
lstart += len;
ix++;
}
cachedPos = pos;
cachedIx = ix;
cachedLstart = lstart;
}
public override string GetString(int beg, int end)
{
if (beg >= maxPos || end <= beg) return "";
int endIx, begIx, endLineStart, begLineStart;
findIndex(beg, out begIx, out begLineStart);
int begCol = beg - begLineStart;
findIndex(end, out endIx, out endLineStart);
int endCol = end - endLineStart;
string s = line[begIx];
if (begIx == endIx) {
// the usual case, substring all on one line
return (endCol <= s.Length)?
s.Substring(begCol, endCol-begCol)
: s.Substring(begCol) + "\n";
}
// the string spans multiple lines, yuk!
StringBuilder sb = new StringBuilder();
if (begCol < s.Length)
sb.Append(s.Substring(begCol));
for( ; ; ) {
sb.Append("\n");
s = line[++begIx];
if (begIx >= endIx) break;
sb.Append(s);
}
if (endCol <= s.Length) {
sb.Append(s.Substring(0, endCol));
} else {
sb.Append(s);
sb.Append("\n");
}
return sb.ToString();
}
public override int Pos
{
get { return cPos; }
set {
cPos = value;
findIndex(cPos, out cLine, out curLineStart);
curLine = line[cLine];
curLineEnd = curLineStart+curLine.Length;
}
}
public override int ReadPos { get { return cPos - 1; } }
}
// ==============================================================
// =========== Nested class StreamBuff : byte files =============
// ==============================================================
/// <summary>
/// This is the Buffer for byte files. It returns 8-bit characters.
/// </summary>
public class StreamBuff : ScanBuff
{
protected BufferedStream bStrm; // input buffer
protected int delta = 1; // number of bytes in chr, could be 0 for EOF.
public StreamBuff(Stream str) { this.bStrm = new BufferedStream(str); }
/// <summary>
/// Read returns subset [0 .. 0xFF] in this application.
/// </summary>
/// <returns>Unicode point as an int [0 .. 0xFF]; -1 for EOF</returns>
public override int Read() {
int ch0 = bStrm.ReadByte();
delta = (ch0 == EOF ? 0 : 1);
return ch0;
}
public override int ReadPos {
get { return (int)bStrm.Position - delta; }
}
public override int Peek()
{
int rslt = bStrm.ReadByte();
bStrm.Seek(-delta, SeekOrigin.Current);
return rslt;
}
public override string GetString(int beg, int end)
{
if (end - beg <= 0) return "";
long savePos = bStrm.Position;
char[] arr = new char[end - beg];
bStrm.Position = (long)beg;
for (int i = 0; i < (end - beg); i++)
arr[i] = (char)bStrm.ReadByte();
bStrm.Position = savePos;
return new String(arr);
}
// Pos is the position *after* reading chr!
public override int Pos
{
get { return (int)bStrm.Position; }
set { bStrm.Position = value; }
}
public static int GetCodePage(string command)
{
command = command.ToLower();
if (command.StartsWith("codepage:"))
command = command.Substring(9);
try
{
if (command.Equals("raw"))
return -1;
else if (command.Equals("guess"))
return -2;
else if (command.Equals("default"))
return 0;
else if (char.IsDigit(command[0]))
return int.Parse(command);
else
{
Encoding enc = Encoding.GetEncoding(command);
return enc.CodePage;
}
}
catch
{
Console.Error.WriteLine(
"Unknown codepage \"{0}\", using machine default", command);
return 0;
}
}
}
#if !BYTEMODE
// ==============================================================
// ===== Nested class CodePageBuff : for unicode text files =====
// ========= Byte files mapped by a specified CodePage ==========
// ==============================================================
public sealed class CodePageBuff : StreamBuff
{
char[] map = new char[256];
public CodePageBuff(Stream stream) : this(stream, Encoding.Default) { }
public CodePageBuff(Stream stream, Encoding enc) : base(stream)
{
bool done;
int bNum, cNum;
byte[] bArray = new byte[256];
if (!enc.IsSingleByte)
throw new NotImplementedException(
"Only UTF-8, UTF-16 and single-byte code pages allowed");
Decoder decoder = enc.GetDecoder();
for (int i = 0; i < 256; i++)
bArray[i] = (byte)i;
decoder.Convert(bArray, 0, 256, this.map, 0, 256, true, out bNum, out cNum, out done);
}
/// <summary>
/// Read returns subset unicode char corresponding to the
/// given character in the chosen codepage.
/// </summary>
/// <returns>Unicode point as an int; -1 for EOF</returns>
public override int Read() {
int ch0 = bStrm.ReadByte();
if (ch0 == EOF)
{
delta = 0; return -1;
}
else
{
delta = 1; return this.map[ch0];
}
}
public override string GetString(int beg, int end)
{
if (end - beg <= 0) return "";
long savePos = bStrm.Position;
int saveDelta = delta;
char[] arr = new char[end - beg];
bStrm.Position = (long)beg;
for (int i = 0; i < (end - beg); i++)
arr[i] = (char)Read();
bStrm.Position = savePos;
delta = saveDelta;
return new String(arr);
}
}
// ==============================================================
// ====== Nested class TextBuff : for unicode text files ========
// ==== This is the UTF8 class, UTF16 handled by subclasses =====
// ==============================================================
/// <summary>
/// This is the Buffer for UTF8 files.
/// It attempts to read the encoding preamble, which for
/// this encoding should be unicode point \uFEFF which is
/// encoded as EF BB BF
/// </summary>
public class TextBuff : ScanBuff
{
protected BufferedStream bStrm; // input buffer
protected int delta = 1; // length of chr, zero for EOF!
/// <summary>
/// TextBuff factory. Reads the file preamble
/// and returns a TextBuff, LittleEndTextBuff or
/// BigEndTextBuff according to the result.
/// </summary>
/// <param name="strm">The underlying stream</param>
/// <returns></returns>
public static ScanBuff NewTextBuff(Stream strm, int fallbackCodepage)
{
// First check if this is a UTF16 file
//
int b0 = strm.ReadByte();
int b1 = strm.ReadByte();
if (b0 == 0xfe && b1 == 0xff)
return new BigEndTextBuff(strm);
if (b0 == 0xff && b1 == 0xfe)
return new LittleEndTextBuff(strm);
int b2 = strm.ReadByte();
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
return new TextBuff(strm);
//
// There is no unicode preamble, so
// we go back to the bytefile default.
//
strm.Seek(0, SeekOrigin.Begin);
switch (fallbackCodepage)
{
case -2:
{
int guess = new Guesser.Scanner(strm).GuessCodepage();
strm.Seek(0, SeekOrigin.Begin);
if (guess == 0)
return new CodePageBuff(strm);
else if (guess == 65001)
return new TextBuff(strm);
else
return new StreamBuff(strm);
}
case -1:
return new StreamBuff(strm);
case 0:
return new CodePageBuff(strm); // Default codepage
case 65001:
return new TextBuff(strm);
case 1200:
return new LittleEndTextBuff(strm);
case 1201:
return new BigEndTextBuff(strm);
default:
return new CodePageBuff(strm, Encoding.GetEncoding(fallbackCodepage));
}
}
protected TextBuff(Stream str) {
this.bStrm = new BufferedStream(str);
}
/// <summary>
/// Read returns UTF32 in this application.
/// </summary>
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
public override int Read()
{
int ch0 = bStrm.ReadByte();
int ch1, ch2, ch3,ch4, ch5;
int result = UnicodeReplacementChar;
if (ch0 < 0x7f)
{
delta = (ch0 == EOF ? 0 : 1);
result = ch0;
}
else if ((ch0 & 0xe0) == 0xc0)
{
delta = 2;
ch1 = bStrm.ReadByte();
if ((ch1 & 0xc0) == 0x80)
result = ((ch0 & 0x1f) << 6) + (ch1 & 0x3f);
}
else if ((ch0 & 0xf0) == 0xe0)
{
delta = 3;
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte();
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80)
result = ((ch0 & 0xf) << 12) + ((ch1 & 0x3f) << 6) + (ch2 & 0x3f);
}
else if ((ch0 & 0xf8) == 0xf0)
{
delta = 4;
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte(); ch3 = bStrm.ReadByte();
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && (ch3 & 0xc0) == 0x80)
{
result = ((ch0 & 0x7) << 18) + ((ch1 & 0x3f) << 12) +
((ch2 & 0x3f) << 6) + (ch3 & 0x3f);
}
}
else if ((ch0 & 0xfc) == 0xf8)
{
delta = 5;
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte();
ch3 = bStrm.ReadByte(); ch4 = bStrm.ReadByte();
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && (ch3 & 0xc0) == 0x80 && (ch4 & 0xc0) == 0x80)
{
result = ((ch0 & 0x3) << 24) + ((ch1 & 0x3f) << 18) +
((ch2 & 0x3f) << 12) + ((ch3 & 0x3f) << 6) + (ch4 & 0x3f);
}
}
else if ((ch0 & 0xfe) == 0xfc)
{
delta = 6;
ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte(); ch3 = bStrm.ReadByte();
ch4 = bStrm.ReadByte(); ch5 = bStrm.ReadByte();
if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 &&
(ch3 & 0xc0) == 0x80 && (ch4 & 0xc0) == 0x80 && (ch5 & 0xc0) == 0x80)
{
result = ((ch0 & 0x1) << 28) + ((ch1 & 0x3f) << 24) + ((ch2 & 0x3f) << 18) +
((ch3 & 0x3f) << 12) + ((ch4 & 0x3f) << 6) + (ch5 & 0x3f);
}
}
return result;
}
public sealed override int ReadPos
{
get { return (int)bStrm.Position - delta; }
}
public sealed override int Peek()
{
int rslt = Read();
bStrm.Seek(-delta, SeekOrigin.Current);
return rslt;
}
/// <summary>
/// Returns the string from the buffer between
/// the given file positions. This needs to be
/// done carefully, as the number of characters
/// is, in general, not equal to (end - beg).
/// </summary>
/// <param name="beg">Begin filepos</param>
/// <param name="end">End filepos</param>
/// <returns></returns>
public override string GetString(int beg, int end)
{
int i;
if (end - beg <= 0) return "";
long savePos = bStrm.Position;
int saveDelta = delta;
//
// The length of the array cannot be larger than
// end - beg, since if the characters expand by
// using surrogate pairs the UTF8 must have taken
// up at least four bytes for any such character.
//
char[] arr = new char[end - beg];
bStrm.Position = (long)beg;
// Normally each call of Read() returns a valid UTF16
// value which will take up one place in the string.
// However, if read returns a value > 0xFFFF then a
// *pair* of surrogate characters must be added.
for (i = 0; bStrm.Position < end; i++)
{
int value = Read();
if (value < 0xFFFF)
arr[i] = (char)value;
else
{
int temp = value - 0x10000;
arr[i++] = (char)(0xD800 + temp / 1024);
arr[i] = (char)(0xDC00 + temp % 1024);
}
}
bStrm.Position = savePos;
delta = saveDelta;
return new String(arr, 0, i);
}
// Pos is the position *after* reading chr!
public sealed override int Pos
{
get { return (int)bStrm.Position; }
set { bStrm.Position = value; }
}
}
// ==============================================================
// ====================== Nested class ==========================
// ==============================================================
/// <summary>
/// This is the Buffer for Big-endian UTF16 files.
/// </summary>
public sealed class BigEndTextBuff : TextBuff
{
internal BigEndTextBuff(Stream str) : base(str) { } //
private int Read16()
{
int ch0 = bStrm.ReadByte();
int ch1 = bStrm.ReadByte();
if (ch1 == EOF)
{
// An EOF in either byte counts as an EOF
delta = (ch0 == EOF ? 0 : 1);
return -1;
}
else
{
delta = 2;
return (ch0 << 8) + ch1;
}
}
/// <summary>
/// Read returns UTF32 in this application.
/// </summary>
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
public override int Read()
{
int utf16 = Read16();
if (utf16 < 0xD800 || utf16 > 0xDBFF)
return utf16;
else
{
int low16 = Read16();
delta = 4;
if (low16 < 0xDC00 || low16 > 0xDFFF)
return UnicodeReplacementChar;
else
return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
}
}
/// <summary>
/// Returns the string from the buffer between
/// the given file positions. This needs to be
/// done carefully, as the number of characters
/// is, in general, not equal to (end - beg).
/// </summary>
/// <param name="beg">Begin filepos</param>
/// <param name="end">End filepos</param>
/// <returns></returns>
public sealed override string GetString(int beg, int end)
{
int i;
if (end - beg <= 0) return "";
long savePos = bStrm.Position;
int saveDelta = delta;
char[] arr = new char[end - beg];
bStrm.Position = (long)beg;
// This stream will have utf16 valid for strings
for (i = 0; bStrm.Position < end; i++)
arr[i] = (char)Read();
bStrm.Position = savePos;
delta = saveDelta;
return new String(arr, 0, i);
}
}
// ==============================================================
// ====================== Nested class ==========================
// ==============================================================
/// <summary>
/// This is the Buffer for Little-endian UTF16 files.
/// </summary>
public sealed class LittleEndTextBuff : TextBuff
{
internal LittleEndTextBuff(Stream str) : base(str) { } // { this.bStrm = new BufferedStream(str); }
private int Read16()
{
int ch0 = bStrm.ReadByte();
int ch1 = bStrm.ReadByte();
if (ch1 == EOF)
{
// An EOF in either byte counts as an EOF
delta = (ch0 == EOF ? 0 : 1);
return -1;
}
else
{
delta = 2;
return (ch1 << 8) + ch0;
}
}
/// <summary>
/// Read returns UTF32 in this application.
/// </summary>
/// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
public override int Read()
{
int utf16 = Read16();
if (utf16 < 0xD800 || utf16 > 0xDBFF)
return utf16;
else
{
int low16 = Read16();
delta = 4;
if (low16 < 0xDC00 || low16 > 0xDFFF)
return UnicodeReplacementChar;
else
return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
}
}
/// <summary>
/// Returns the string from the buffer between
/// the given file positions. This needs to be
/// done carefully, as the number of characters
/// is, in general, not equal to (end - beg).
/// </summary>
/// <param name="beg">Begin filepos</param>
/// <param name="end">End filepos</param>
/// <returns></returns>
public sealed override string GetString(int beg, int end)
{
int i;
if (end - beg <= 0) return "";
long savePos = bStrm.Position;
int saveDelta = delta;
char[] arr = new char[end - beg];
bStrm.Position = (long)beg;
// This stream will have utf16 valid for strings
for (i = 0; bStrm.Position < end; i++)
arr[i] = (char)Read16();
bStrm.Position = savePos;
delta = saveDelta;
return new String(arr, 0, i);
}
}
#endif // !BYTEMODE
#endif // !NOFILES
#endregion Buffer classes
// =================== End Nested classes =======================
#if !NOFILES
public Scanner(Stream file) {
SetSource(file); // no unicode option
}
public Scanner(Stream file, string codepage) {
SetSource(file); // Not unicode, ignoring codepage
}
#endif // !NOFILES
public Scanner() { }
void GetChr()
{
if (chr == '\n') // This needs to be fixed for other conventions
{
lineStartNum = cNum + 1;
lNum++;
}
chr = buffer.Read();
cNum++;
}
void MarkToken()
{
tokPos = buffer.ReadPos;
tokNum = cNum;
tokLin = lNum;
tokCol = cNum - lineStartNum;
}
void MarkEnd()
{
tokTxt = null;
tokLen = cNum - tokNum;
tokEPos = buffer.ReadPos;
tokELin = lNum;
tokECol = cNum - lineStartNum;
}
// ==============================================================
// ===== Initialization of string-based input buffers ====
// ==============================================================
/// <summary>
/// Create and initialize a StringBuff buffer object for this scanner
/// </summary>
/// <param name="source">the input string</param>
/// <param name="offset">starting offset in the string</param>
public void SetSource(string source, int offset)
{
this.buffer = new StringBuff(source);
this.buffer.Pos = offset;
this.lNum = 0;
this.cNum = offset - 1;
this.chr = '\n'; // to initialize yyline, yycol and lineStart
GetChr();
}
#if !NOFILES
// ================ LineBuffer Initialization ===================
/// <summary>
/// Create and initialize a LineBuff buffer object for this scanner
/// </summary>
/// <param name="source">the list of input strings</param>
public void SetSource(IList<string> source)
{
this.buffer = new LineBuff(source);
this.chr = '\n'; // to initialize yyline, yycol and lineStart
this.lNum = 0;
this.cNum = -1;
GetChr();
}
// =============== StreamBuffer Initialization ==================
/// <summary>
/// Create and initialize a StreamBuff buffer object for this scanner.
/// StreamBuff is buffer for 8-bit byte files.
/// </summary>
/// <param name="source">the input byte stream</param>
public void SetSource(Stream source)
{
this.buffer = new StreamBuff(source);
this.lNum = 0;
this.cNum = -1;
this.chr = '\n'; // to initialize yyline, yycol and lineStart
GetChr();
}
#if !BYTEMODE
// ================ TextBuffer Initialization ===================
/// <summary>
/// Create and initialize a TextBuff buffer object for this scanner.
/// TextBuff is a buffer for encoded unicode files.
/// </summary>
/// <param name="source">the input text file</param>
/// <param name="fallbackCodepage">Codepage to use if file has
/// no BOM. For 0, use machine default; for -1, 8-bit binary</param>
public void SetSource(Stream source, int fallbackCodepage)
{
this.buffer = TextBuff.NewTextBuff(source, fallbackCodepage);
this.lNum = 0;
this.cNum = -1;
this.chr = '\n'; // to initialize yyline, yycol and lineStart
GetChr();
}
#endif // !BYTEMODE
#endif // !NOFILES
// ==============================================================
#if BABEL
//
// Get the next token for Visual Studio
//
// "state" is the inout mode variable that maintains scanner
// state between calls, using the EolState property. In principle,
// if the calls of EolState are costly set could be called once
// only per line, at the start; and get called only at the end
// of the line. This needs more infrastructure ...
//
public int GetNext(ref int state, out int start, out int end)
{
Tokens next;
int s, e;
s = state; // state at start
EolState = state;
next = (Tokens)Scan();
state = EolState;
e = state; // state at end;
start = tokPos;
end = tokEPos - 1; // end is the index of last char.
return (int)next;
}
#endif // BABEL
// ======== IScanner<> Implementation =========
public override int yylex()
{
// parserMax is set by reflecting on the Tokens
// enumeration. If maxParseToken is defined
// that is used, otherwise int.MaxValue is used.
int next;
do { next = Scan(); } while (next >= parserMax);
return next;
}
int yyleng { get { return tokLen; } }
int yypos { get { return tokPos; } }
int yyline { get { return tokLin; } }
int yycol { get { return tokCol; } }
public string yytext
{
get
{
if (tokTxt == null)
tokTxt = buffer.GetString(tokPos, tokEPos);
return tokTxt;
}
}
void yyless(int n) {
buffer.Pos = tokPos;
// Must read at least one char, so set before start.
cNum = tokNum - 1; GetChr();
// Now ensure that line counting is correct.
lNum = tokLin;
// And count the rest of the text.
for (int i = 0; i < n; i++) GetChr();
MarkEnd();
}
// ============ methods available in actions ==============
internal int YY_START {
get { return currentScOrd; }
set { currentScOrd = value;
currentStart = startState[value];
}
}
internal void BEGIN(int next) {
currentScOrd = next;
currentStart = startState[next];
}
// ============== The main tokenizer code =================
int Scan()
{
try {
for (; ; )
{
int next; // next state to enter
#if BACKUP
Result rslt = Result.noMatch;
#endif // BACKUP
#if LEFTANCHORS
for (;;)
{
// Discard characters that do not start any pattern.
// Must check the left anchor condition after *every* GetChr!
state = (lineStartNum == cNum ? anchorState[currentScOrd] : currentStart);
if ((next = NextState()) != goStart)
break; // LOOP EXIT HERE...
GetChr();
}
#else // !LEFTANCHORS
state = currentStart;
while ((next = NextState()) == goStart)
// At this point, the current character has no
// transition from the current state. We discard
// the "no-match" char. In traditional LEX such
// characters are echoed to the console.
GetChr();
#endif // LEFTANCHORS
// At last, a valid transition ...
MarkToken();
state = next;
GetChr();
while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum
#if BACKUP
if (state <= maxAccept && next > maxAccept) // need to prepare backup data
{
// ctx is an object. The fields may be
// mutated by the call to Recurse2.
// On return the data in ctx is the
// *latest* accept state that was found.
Context ctx = new Context();
rslt = Recurse2(ctx, next);
if (rslt == Result.noMatch)
RestoreStateAndPos(ctx);
break;
}
else
#endif // BACKUP
{
state = next;
GetChr();
}
if (state <= maxAccept)
{
MarkEnd();
#region ActionSwitch
#pragma warning disable 162
switch (state)
{
case eofNum:
if (yywrap())
return (int)Tokens.EOF;
break;
case 1:
;
break;
case 2:
case 3:
return (int)Tokens.ENDLINE;
break;
case 4:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.LPAREN;
break;
case 5:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.RPAREN;
break;
case 6:
yylval._op_type_node = PT.create_op_type_node(Operators.Multiplication, yylloc);
return (int)Tokens.MULT;
break;
case 7:
yylval._op_type_node = PT.create_op_type_node(Operators.Plus, yylloc);
return (int)Tokens.PLUS;
break;
case 8:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.COLUMN;
break;
case 9:
yylval._op_type_node = PT.create_op_type_node(Operators.Minus, yylloc);
return (int)Tokens.MINUS;
break;
case 10:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.DOT;
break;
case 11:
yylval._op_type_node = PT.create_op_type_node(Operators.Division, yylloc);
return (int)Tokens.DIVIDE;
break;
case 12:
yylval._const_node = PT.create_int_const(yytext, yylloc);
return (int)Tokens.INTNUM;
break;
case 13:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.COLON;
break;
case 14:
return (int)Tokens.SEMICOLUMN;
break;
case 15:
yylval._op_type_node = PT.create_op_type_node(Operators.Less, yylloc);
return (int)Tokens.LT;
break;
case 16:
yylval._op_type_node = PT.create_op_type_node(Operators.Assignment, yylloc);
return (int)Tokens.ASSIGN;
break;
case 17:
yylval._op_type_node = PT.create_op_type_node(Operators.Greater, yylloc);
return (int)Tokens.GT;
break;
case 18:
case 19:
case 20:
case 23:
case 24:
case 25:
case 27:
case 29:
case 31:
case 32:
case 34:
case 35:
case 36:
if (ScannerHelper.keywords.ContainsKey(yytext))
{
yylval._token_info = PT.create_token_info(yytext, yylloc);
return ScannerHelper.keywords[yytext];
}
else
{
yylval._ident = PT.create_directive_name(yytext, yylloc);
return (int)Tokens.ID;
}
break;
case 21:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.LSQUARE;
break;
case 22:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.RSQUARE;
break;
case 26:
yylval._op_type_node = PT.create_op_type_node(Operators.LogicalOR, yylloc);
return (int)Tokens.OR;
break;
case 28:
yylval._op_type_node = PT.create_op_type_node(Operators.LogicalNOT, yylloc);
return (int)Tokens.NOT;
break;
case 30:
yylval._op_type_node = PT.create_op_type_node(Operators.LogicalAND, yylloc);
return (int)Tokens.AND;
break;
case 33:
yylval._const_node = PT.create_bool_const(true, yylloc);
return (int)Tokens.TRUECONST;
break;
case 37:
yylval._const_node = PT.create_bool_const(false, yylloc);
return (int)Tokens.FALSECONST;
break;
case 38:
yylval._op_type_node = PT.create_op_type_node(Operators.GreaterEqual, yylloc);
return (int)Tokens.GE;
break;
case 39:
yylval._op_type_node = PT.create_op_type_node(Operators.Equal, yylloc);
return (int)Tokens.EQ;
break;
case 40:
yylval._op_type_node = PT.create_op_type_node(Operators.LessEqual, yylloc);
return (int)Tokens.LE;
break;
case 41:
yylval._const_node = PT.create_double_const(yytext, yylloc);
return (int)Tokens.FLOATNUM;
break;
case 42:
yylval._token_info = PT.create_token_info(yytext, yylloc);
return (int)Tokens.FUNCRETSYMB;
break;
case 43:
case 45:
case 48:
yylval._const_node = PT.create_string_const(yytext, yylloc);
return (int)Tokens.STRINGLITERAL;
break;
case 44:
yylval._const_node = PT.create_string_const(yytext, yylloc);
return (int)Tokens.MULTILINESTRING;
break;
case 46:
case 47:
BEGIN(INITIAL);
break;
case 49:
yylval._op_type_node = PT.create_op_type_node(Operators.NotEqual, yylloc);
return (int)Tokens.NE;
break;
default:
break;
}
#pragma warning restore 162
#endregion
}
}
} // end try
finally {
// User-specified epilog to scan()
yylloc = new LexLocation(tokLin, tokCol, tokELin, tokECol);
// End, user-specified epilog
} // end finally
}
#if BACKUP
Result Recurse2(Context ctx, int next)
{
// Assert: at entry "state" is an accept state AND
// NextState(state, chr) != goStart AND
// NextState(state, chr) is not an accept state.
//
SaveStateAndPos(ctx);
state = next;
// if (state == eofNum) return Result.accept;
GetChr();
while ((next = NextState()) > eofNum)
{
if (state <= maxAccept && next > maxAccept) // need to update backup data
SaveStateAndPos(ctx);
state = next;
if (state == eofNum) return Result.accept;
GetChr();
}
return (state <= maxAccept ? Result.accept : Result.noMatch);
}
void SaveStateAndPos(Context ctx)
{
ctx.bPos = buffer.Pos;
ctx.cNum = cNum;
ctx.lNum = lNum;
ctx.state = state;
ctx.cChr = chr;
}
void RestoreStateAndPos(Context ctx)
{
buffer.Pos = ctx.bPos;
cNum = ctx.cNum;
lNum = ctx.lNum;
state = ctx.state;
chr = ctx.cChr;
}
void RestorePos(Context ctx) { buffer.Pos = ctx.bPos; cNum = ctx.cNum; }
#endif // BACKUP
// ============= End of the tokenizer code ================
#if STACK
internal void yy_clear_stack() { scStack.Clear(); }
internal int yy_top_state() { return scStack.Peek(); }
internal void yy_push_state(int state)
{
scStack.Push(currentScOrd);
BEGIN(state);
}
internal void yy_pop_state()
{
// Protect against input errors that pop too far ...
if (scStack.Count > 0) {
int newSc = scStack.Pop();
BEGIN(newSc);
} // Otherwise leave stack unchanged.
}
#endif // STACK
internal void ECHO() { Console.Out.Write(yytext); }
#region UserCodeSection
class ScannerHelper
{
public static Dictionary<string, int> keywords;
static ScannerHelper()
{
keywords = new Dictionary<string, int>();
keywords.Add("python",(int)Tokens.kINDENT);
keywords.Add("nohtyp",(int)Tokens.kDEDENT);
keywords.Add("if",(int)Tokens.kIF);
keywords.Add("else",(int)Tokens.kELSE);
keywords.Add("while",(int)Tokens.kWHILE);
keywords.Add("and",(int)Tokens.AND);
keywords.Add("or",(int)Tokens.OR);
keywords.Add("not",(int)Tokens.NOT);
keywords.Add("def",(int)Tokens.kDEF);
keywords.Add("fed",(int)Tokens.kFED);
keywords.Add("return",(int)Tokens.kRETURN);
keywords.Add("int", (int)Tokens.bINT);
keywords.Add("float", (int)Tokens.bFLOAT);
keywords.Add("str", (int)Tokens.bSTR);
keywords.Add("bool", (int)Tokens.bBOOL);
}
}
#endregion
} // end class Scanner
} // end namespace
Reference in a new issue Copy permalink