// // Experimental embedded frame // Version 1.1.3 of 18-April-2010 // ## Derived from gplex.frame version of 2-September-2006. ## Code page support for files without a BOM. ## Left and Right Anchored state support. ## Start condition stack. Two generic params. ## Using fixed length context handling for right anchors // ##-->defines using System; using System.IO; using System.Text; using System.Globalization; using System.Collections.Generic; using System.Runtime.Serialization; using System.Diagnostics.CodeAnalysis; ##-->version290 ##-->usingDcl { /// /// Summary Canonical example of GPLEX automaton /// #if STANDALONE // // These are the dummy declarations for stand-alone GPLEX applications // normally these declarations would come from the parser. // If you declare /noparser, or %option noparser then you get this. // ##-->translate $public enum $Tokens { EOF = 0, maxParseToken = int.MaxValue // must have at least these two, values are almost arbitrary } ##-->translate $public abstract class $ScanBase { [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")] public abstract int yylex(); [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yywrap")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yywrap")] protected virtual bool yywrap() { return true; } #if BABEL protected abstract int CurrentSc { get; set; } // EolState is the 32-bit of state data persisted at // the end of each line for Visual Studio colorization. // The default is to return CurrentSc. You must override // this if you want more complicated behavior. public virtual int EolState { get { return CurrentSc; } set { CurrentSc = value; } } } ##-->translate $public interface IColorScan { void SetSource(string source, int offset); int GetNext(ref int state, out int start, out int end); #endif // BABEL } #endif // STANDALONE // If the compiler can't find the scanner base class maybe you // need to run GPPG with the /gplex option, or GPLEX with /noparser #if BABEL ##-->translate $public sealed partial class $Scanner : $ScanBase, IColorScan { private ScanBuff buffer; int currentScOrd; // start condition ordinal protected override int CurrentSc { // The current start state is a property // to try to avoid the user error of setting // scState but forgetting to update the FSA // start state "currentStart" // get { return currentScOrd; } // i.e. return YY_START; set { currentScOrd = value; // i.e. BEGIN(value); currentStart = startState[value]; } } #else // BABEL ##-->translate $public sealed partial class $Scanner : $ScanBase { private ScanBuff buffer; int currentScOrd; // start condition ordinal #endif // BABEL /// /// The input buffer for this scanner. /// [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public ScanBuff Buffer { get { return buffer; } } private static int GetMaxParseToken() { ##-->translate System.Reflection.FieldInfo f = typeof($Tokens).GetField("maxParseToken"); return (f == null ? int.MaxValue : (int)f.GetValue(null)); } static int parserMax = GetMaxParseToken(); enum Result {accept, noMatch, contextFound}; ##-->consts #region user code ##-->codeIncl #endregion user code int state; int currentStart = startState[0]; int code; // last code read int cCol; // column number of code int lNum; // current line number // // The following instance variables are used, among other // things, for constructing the yylloc location objects. // int tokPos; // buffer position at start of token int tokCol; // zero-based column number at start of token int tokLin; // line number at start of token int tokEPos; // buffer position at end of token int tokECol; // column number at end of token int tokELin; // line number at end of token string tokTxt; // lazily constructed text of token #if STACK private Stack scStack = new Stack(); #endif // STACK ##-->tableDef #if BACKUP // ============================================================== // == Nested struct used for backup in automata that do backup == // ============================================================== struct Context // class used for automaton backup. { public int bPos; public int rPos; // scanner.readPos saved value public int cCol; public int lNum; // Need this in case of backup over EOL. public int state; public int cChr; } private Context ctx = new Context(); #endif // BACKUP // ============================================================== // ==== Nested struct to support input switching in scanners ==== // ============================================================== struct BufferContext { internal ScanBuff buffSv; internal int chrSv; internal int cColSv; internal int lNumSv; } // ============================================================== // ===== Private methods to save and restore buffer contexts ==== // ============================================================== /// /// This method creates a buffer context record from /// the current buffer object, together with some /// scanner state values. /// [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] BufferContext MkBuffCtx() { BufferContext rslt; rslt.buffSv = this.buffer; rslt.chrSv = this.code; rslt.cColSv = this.cCol; rslt.lNumSv = this.lNum; return rslt; } /// /// This method restores the buffer value and allied /// scanner state from the given context record value. /// [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void RestoreBuffCtx(BufferContext value) { this.buffer = value.buffSv; this.code = value.chrSv; this.cCol = value.cColSv; this.lNum = value.lNumSv; } // =================== End Nested classes ======================= #if !NOFILES ##-->translate $public $Scanner(Stream file) { ##-->bufferCtor } #endif // !NOFILES ##-->translate $public $Scanner() { } private int readPos; void GetCode() { if (code == '\n') // This needs to be fixed for other conventions // i.e. [\r\n\205\u2028\u2029] { cCol = -1; lNum++; } readPos = buffer.Pos; // Now read new codepoint. code = buffer.Read(); if (code > ScanBuff.EndOfFile) { #if (!BYTEMODE) if (code >= 0xD800 && code <= 0xDBFF) { int next = buffer.Read(); if (next < 0xDC00 || next > 0xDFFF) code = ScanBuff.UnicodeReplacementChar; else code = (0x10000 + (code & 0x3FF << 10) + (next & 0x3FF)); } #endif cCol++; } } void MarkToken() { #if (!PERSIST) buffer.Mark(); #endif tokPos = readPos; tokLin = lNum; tokCol = cCol; } void MarkEnd() { tokTxt = null; tokEPos = readPos; tokELin = lNum; tokECol = cCol; } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int Peek() { int rslt, codeSv = code, cColSv = cCol, lNumSv = lNum, bPosSv = buffer.Pos; GetCode(); rslt = code; lNum = lNumSv; cCol = cColSv; code = codeSv; buffer.Pos = bPosSv; return rslt; } // ============================================================== // ===== Initialization of string-based input buffers ==== // ============================================================== /// /// Create and initialize a StringBuff buffer object for this scanner /// /// the input string /// starting offset in the string [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(string source, int offset) { this.buffer = ScanBuff.GetBuffer(source); this.buffer.Pos = offset; this.lNum = 0; this.code = '\n'; // to initialize yyline, yycol and lineStart GetCode(); } #if !NOFILES // ================ LineBuffer Initialization =================== /// /// Create and initialize a LineBuff buffer object for this scanner /// /// the list of input strings [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(IList source) { this.buffer = ScanBuff.GetBuffer(source); this.code = '\n'; // to initialize yyline, yycol and lineStart this.lNum = 0; GetCode(); } // =============== StreamBuffer Initialization ================== /// /// Create and initialize a StreamBuff buffer object for this scanner. /// StreamBuff is buffer for 8-bit byte files. /// /// the input byte stream [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(Stream source) { this.buffer = ScanBuff.GetBuffer(source); this.lNum = 0; this.code = '\n'; // to initialize yyline, yycol and lineStart GetCode(); } #if !BYTEMODE // ================ TextBuffer Initialization =================== /// /// Create and initialize a TextBuff buffer object for this scanner. /// TextBuff is a buffer for encoded unicode files. /// /// the input text file /// Code page to use if file has /// no BOM. For 0, use machine default; for -1, 8-bit binary [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(Stream source, int fallbackCodePage) { this.buffer = ScanBuff.GetBuffer(source, fallbackCodePage); this.lNum = 0; this.code = '\n'; // to initialize yyline, yycol and lineStart GetCode(); } #endif // !BYTEMODE #endif // !NOFILES // ============================================================== #if BABEL // // Get the next token for Visual Studio // // "state" is the inout mode variable that maintains scanner // state between calls, using the EolState property. In principle, // if the calls of EolState are costly set could be called once // only per line, at the start; and get called only at the end // of the line. This needs more infrastructure ... // public int GetNext(ref int state, out int start, out int end) { ##-->translate $Tokens next; int s, e; s = state; // state at start EolState = state; ##-->translate next = ($Tokens)Scan(); state = EolState; e = state; // state at end; start = tokPos; end = tokEPos - 1; // end is the index of last char. return (int)next; } #endif // BABEL // ======== AbstractScanner<> Implementation ========= [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")] public override int yylex() { // parserMax is set by reflecting on the Tokens // enumeration. If maxParseToken is defined // that is used, otherwise int.MaxValue is used. int next; do { next = Scan(); } while (next >= parserMax); return next; } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int yypos { get { return tokPos; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int yyline { get { return tokLin; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int yycol { get { return tokCol; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yytext")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yytext")] public string yytext { get { if (tokTxt == null) tokTxt = buffer.GetString(tokPos, tokEPos); return tokTxt; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void yyless(int n) { buffer.Pos = tokPos; // Must read at least one char, so set before start. cCol = tokCol - 1; GetCode(); // Now ensure that line counting is correct. lNum = tokLin; // And count the rest of the text. for (int i = 0; i < n; i++) GetCode(); MarkEnd(); } // // It would be nice to count backward in the text // but it does not seem possible to re-establish // the correct column counts except by going forward. // [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void _yytrunc(int n) { yyless(yyleng - n); } // // This is painful, but we no longer count // codepoints. For the overwhelming majority // of cases the single line code is fast, for // the others, well, at least it is all in the // buffer so no files are touched. Note that we // can't use (tokEPos - tokPos) because of the // possibility of surrogate pairs in the token. // [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyleng")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyleng")] public int yyleng { get { #if BYTEMODE return tokEPos - tokPos; #else if (tokELin == tokLin) return tokECol - tokCol; else { int ch; int count = 0; int save = buffer.Pos; buffer.Pos = tokPos; do { ch = buffer.Read(); if (!char.IsHighSurrogate((char)ch)) count++; } while (buffer.Pos < tokEPos && ch != ScanBuff.EndOfFile); buffer.Pos = save; return count; } #endif // BYTEMODE } } // ============ methods available in actions ============== [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal int YY_START { get { return currentScOrd; } set { currentScOrd = value; currentStart = startState[value]; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal void BEGIN(int next) { currentScOrd = next; currentStart = startState[next]; } // ============== The main tokenizer code ================= int Scan() { ##-->prolog for (; ; ) { int next; // next state to enter #if BACKUP Result rslt = Result.noMatch; #endif // BACKUP #if LEFTANCHORS for (;;) { // Discard characters that do not start any pattern. // Must check the left anchor condition after *every* GetCode! state = ((cCol == 0) ? anchorState[currentScOrd] : currentStart); if ((next = NextState()) != goStart) break; // LOOP EXIT HERE... GetCode(); } #else // !LEFTANCHORS state = currentStart; while ((next = NextState()) == goStart) // At this point, the current character has no // transition from the current state. We discard // the "no-match" char. In traditional LEX such // characters are echoed to the console. GetCode(); #endif // LEFTANCHORS // At last, a valid transition ... MarkToken(); state = next; GetCode(); while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum #if BACKUP if (state <= maxAccept && next > maxAccept) // need to prepare backup data { // ctx is an object. The fields may be // mutated by the call to Recurse2. // On return the data in ctx is the // *latest* accept state that was found. rslt = Recurse2(ref ctx, next); if (rslt == Result.noMatch) RestoreStateAndPos(ref ctx); break; } else #endif // BACKUP { state = next; GetCode(); } if (state <= maxAccept) { MarkEnd(); ##-->actionCases } } ##-->epilog } #if BACKUP Result Recurse2(ref Context ctx, int next) { // Assert: at entry "state" is an accept state AND // NextState(state, code) != goStart AND // NextState(state, code) is not an accept state. // SaveStateAndPos(ref ctx); state = next; GetCode(); while ((next = NextState()) > eofNum) { if (state <= maxAccept && next > maxAccept) // need to update backup data SaveStateAndPos(ref ctx); state = next; if (state == eofNum) return Result.accept; GetCode(); } return (state <= maxAccept ? Result.accept : Result.noMatch); } void SaveStateAndPos(ref Context ctx) { ctx.bPos = buffer.Pos; ctx.rPos = readPos; ctx.cCol = cCol; ctx.lNum = lNum; ctx.state = state; ctx.cChr = code; } void RestoreStateAndPos(ref Context ctx) { buffer.Pos = ctx.bPos; readPos = ctx.rPos; cCol = ctx.cCol; lNum = ctx.lNum; state = ctx.state; code = ctx.cChr; } #endif // BACKUP // ============= End of the tokenizer code ================ #if STACK [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal void yy_clear_stack() { scStack.Clear(); } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal int yy_top_state() { return scStack.Peek(); } internal void yy_push_state(int state) { scStack.Push(currentScOrd); BEGIN(state); } internal void yy_pop_state() { // Protect against input errors that pop too far ... if (scStack.Count > 0) { int newSc = scStack.Pop(); BEGIN(newSc); } // Otherwise leave stack unchanged. } #endif // STACK [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal void ECHO() { Console.Out.Write(yytext); } ##-->userCode } // end class $Scanner ##-->embeddedBuffers } // end namespace