// // This CSharp output file generated by Gardens Point LEX // Version: 1.1.3.301 // Machine: DESKTOP-G8V08V4 // DateTime: 27.03.2019 21:35:17 // UserName: ????????? // GPLEX input file // GPLEX frame file // // Option settings: unicode, parser, minimize // Option settings: classes, compressMap, compressNext, persistBuffer, embedbuffers // Fallback code page: Target machine default // // // Experimental embedded frame // Version 1.1.3 of 18-April-2010 // // #define BACKUP #define PERSIST using System; using System.IO; using System.Text; using System.Globalization; using System.Collections.Generic; using System.Runtime.Serialization; using System.Diagnostics.CodeAnalysis; using PascalABCCompiler.SyntaxTree; using QUT.Gppg; namespace GPPGPreprocessor3 { /// /// Summary Canonical example of GPLEX automaton /// #if STANDALONE // // These are the dummy declarations for stand-alone GPLEX applications // normally these declarations would come from the parser. // If you declare /noparser, or %option noparser then you get this. // public enum Tokens { EOF = 0, maxParseToken = int.MaxValue // must have at least these two, values are almost arbitrary } public abstract class ScanBase { [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")] public abstract int yylex(); [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yywrap")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yywrap")] protected virtual bool yywrap() { return true; } #if BABEL protected abstract int CurrentSc { get; set; } // EolState is the 32-bit of state data persisted at // the end of each line for Visual Studio colorization. // The default is to return CurrentSc. You must override // this if you want more complicated behavior. public virtual int EolState { get { return CurrentSc; } set { CurrentSc = value; } } } public interface IColorScan { void SetSource(string source, int offset); int GetNext(ref int state, out int start, out int end); #endif // BABEL } #endif // STANDALONE // If the compiler can't find the scanner base class maybe you // need to run GPPG with the /gplex option, or GPLEX with /noparser #if BABEL public sealed partial class PreprocessorScanner : ScanBase, IColorScan { private ScanBuff buffer; int currentScOrd; // start condition ordinal protected override int CurrentSc { // The current start state is a property // to try to avoid the user error of setting // scState but forgetting to update the FSA // start state "currentStart" // get { return currentScOrd; } // i.e. return YY_START; set { currentScOrd = value; // i.e. BEGIN(value); currentStart = startState[value]; } } #else // BABEL public sealed partial class PreprocessorScanner : ScanBase { private ScanBuff buffer; int currentScOrd; // start condition ordinal #endif // BABEL /// /// The input buffer for this scanner. /// [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public ScanBuff Buffer { get { return buffer; } } private static int GetMaxParseToken() { System.Reflection.FieldInfo f = typeof(Tokens).GetField("maxParseToken"); return (f == null ? int.MaxValue : (int)f.GetValue(null)); } static int parserMax = GetMaxParseToken(); enum Result {accept, noMatch, contextFound}; const int maxAccept = 13; const int initial = 14; const int eofNum = 0; const int goStart = -1; const int INITIAL = 0; const int INSIDEDIRECTIVE = 1; const int COMMENT = 2; const int COMMENT1 = 3; const int COMMENTONELINE = 4; #region user code #endregion user code int state; int currentStart = startState[0]; int code; // last code read int cCol; // column number of code int lNum; // current line number // // The following instance variables are used, among other // things, for constructing the yylloc location objects. // int tokPos; // buffer position at start of token int tokCol; // zero-based column number at start of token int tokLin; // line number at start of token int tokEPos; // buffer position at end of token int tokECol; // column number at end of token int tokELin; // line number at end of token string tokTxt; // lazily constructed text of token #if STACK private Stack scStack = new Stack(); #endif // STACK #region ScannerTables struct Table { public int min; public int rng; public int dflt; public sbyte[] nxt; public Table(int m, int x, int d, sbyte[] n) { min = m; rng = x; dflt = d; nxt = n; } }; static int[] startState = new int[] {14, 18, 19, 20, 21, 0}; #region CompressedCharacterMap // // There are 12 equivalence classes // There are 2 character sequence regions // There are 1 tables, 126 entries // There are 1 runs, 0 singletons // Decision tree depth is 1 // static sbyte[] mapC0 = new sbyte[126] { /* '\0' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 2, 4, 4, /* '\x10' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* '\x20' */ 4, 4, 4, 4, 6, 4, 4, 1, 9, 11, 10, 4, 4, 4, 8, 3, /* '0' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* '@' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* 'P' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* '`' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* 'p' */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 8, 7 }; static sbyte MapC(int code) { // '\0' <= code <= '\U0010FFFF' if (code < 126) // '\0' <= code <= '}' return mapC0[code - 0]; else // '~' <= code <= '\U0010FFFF' return (sbyte)4; } #endregion static Table[] NxS = new Table[22] { /* NxS[ 0] */ new Table(0, 0, 0, null), /* NxS[ 1] */ new Table(8, 5, -1, new sbyte[] {1, -1, -1, -1, 1}), /* NxS[ 2] */ new Table(6, 1, -1, new sbyte[] {4}), /* NxS[ 3] */ new Table(0, 0, -1, null), /* NxS[ 4] */ new Table(0, 0, -1, null), /* NxS[ 5] */ new Table(0, 3, 5, new sbyte[] {-1, 5, -1}), /* NxS[ 6] */ new Table(1, 1, -1, new sbyte[] {15}), /* NxS[ 7] */ new Table(7, 1, 7, new sbyte[] {-1}), /* NxS[ 8] */ new Table(0, 0, -1, null), /* NxS[ 9] */ new Table(0, 0, -1, null), /* NxS[ 10] */ new Table(0, 0, -1, null), /* NxS[ 11] */ new Table(0, 0, -1, null), /* NxS[ 12] */ new Table(11, 1, -1, new sbyte[] {13}), /* NxS[ 13] */ new Table(0, 0, -1, null), /* NxS[ 14] */ new Table(8, 10, -1, new sbyte[] {1, 17, -1, -1, 1, 15, -1, 16, -1, 2}), /* NxS[ 15] */ new Table(0, 2, 15, new sbyte[] {-1, 6}), /* NxS[ 16] */ new Table(3, 1, -1, new sbyte[] {5}), /* NxS[ 17] */ new Table(10, 1, -1, new sbyte[] {3}), /* NxS[ 18] */ new Table(7, 1, 7, new sbyte[] {8}), /* NxS[ 19] */ new Table(7, 1, 9, new sbyte[] {10}), /* NxS[ 20] */ new Table(10, 1, 11, new sbyte[] {12}), /* NxS[ 21] */ new Table(0, 0, -1, null), }; int NextState() { if (code == ScanBuff.EndOfFile) return eofNum; else unchecked { int rslt; int idx = MapC(code) - NxS[state].min; if (idx < 0) idx += 12; if ((uint)idx >= (uint)NxS[state].rng) rslt = NxS[state].dflt; else rslt = NxS[state].nxt[idx]; return rslt; } } #endregion #if BACKUP // ============================================================== // == Nested struct used for backup in automata that do backup == // ============================================================== struct Context // class used for automaton backup. { public int bPos; public int rPos; // scanner.readPos saved value public int cCol; public int lNum; // Need this in case of backup over EOL. public int state; public int cChr; } private Context ctx = new Context(); #endif // BACKUP // ============================================================== // ==== Nested struct to support input switching in scanners ==== // ============================================================== struct BufferContext { internal ScanBuff buffSv; internal int chrSv; internal int cColSv; internal int lNumSv; } // ============================================================== // ===== Private methods to save and restore buffer contexts ==== // ============================================================== /// /// This method creates a buffer context record from /// the current buffer object, together with some /// scanner state values. /// [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] BufferContext MkBuffCtx() { BufferContext rslt; rslt.buffSv = this.buffer; rslt.chrSv = this.code; rslt.cColSv = this.cCol; rslt.lNumSv = this.lNum; return rslt; } /// /// This method restores the buffer value and allied /// scanner state from the given context record value. /// [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void RestoreBuffCtx(BufferContext value) { this.buffer = value.buffSv; this.code = value.chrSv; this.cCol = value.cColSv; this.lNum = value.lNumSv; } // =================== End Nested classes ======================= #if !NOFILES public PreprocessorScanner(Stream file) { SetSource(file, 0); // unicode option } public PreprocessorScanner(Stream file, string codepage) { SetSource(file, CodePageHandling.GetCodePage(codepage)); } #endif // !NOFILES public PreprocessorScanner() { } private int readPos; void GetCode() { if (code == '\n') // This needs to be fixed for other conventions // i.e. [\r\n\205\u2028\u2029] { cCol = -1; lNum++; } readPos = buffer.Pos; // Now read new codepoint. code = buffer.Read(); if (code > ScanBuff.EndOfFile) { #if (!BYTEMODE) if (code >= 0xD800 && code <= 0xDBFF) { int next = buffer.Read(); if (next < 0xDC00 || next > 0xDFFF) code = ScanBuff.UnicodeReplacementChar; else code = (0x10000 + (code & 0x3FF << 10) + (next & 0x3FF)); } #endif cCol++; } } void MarkToken() { #if (!PERSIST) buffer.Mark(); #endif tokPos = readPos; tokLin = lNum; tokCol = cCol; } void MarkEnd() { tokTxt = null; tokEPos = readPos; tokELin = lNum; tokECol = cCol; } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int Peek() { int rslt, codeSv = code, cColSv = cCol, lNumSv = lNum, bPosSv = buffer.Pos; GetCode(); rslt = code; lNum = lNumSv; cCol = cColSv; code = codeSv; buffer.Pos = bPosSv; return rslt; } // ============================================================== // ===== Initialization of string-based input buffers ==== // ============================================================== /// /// Create and initialize a StringBuff buffer object for this scanner /// /// the input string /// starting offset in the string [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(string source, int offset) { this.buffer = ScanBuff.GetBuffer(source); this.buffer.Pos = offset; this.lNum = 0; this.code = '\n'; // to initialize yyline, yycol and lineStart GetCode(); } #if !NOFILES // ================ LineBuffer Initialization =================== /// /// Create and initialize a LineBuff buffer object for this scanner /// /// the list of input strings [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(IList source) { this.buffer = ScanBuff.GetBuffer(source); this.code = '\n'; // to initialize yyline, yycol and lineStart this.lNum = 0; GetCode(); } // =============== StreamBuffer Initialization ================== /// /// Create and initialize a StreamBuff buffer object for this scanner. /// StreamBuff is buffer for 8-bit byte files. /// /// the input byte stream [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(Stream source) { this.buffer = ScanBuff.GetBuffer(source); this.lNum = 0; this.code = '\n'; // to initialize yyline, yycol and lineStart GetCode(); } #if !BYTEMODE // ================ TextBuffer Initialization =================== /// /// Create and initialize a TextBuff buffer object for this scanner. /// TextBuff is a buffer for encoded unicode files. /// /// the input text file /// Code page to use if file has /// no BOM. For 0, use machine default; for -1, 8-bit binary [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] public void SetSource(Stream source, int fallbackCodePage) { this.buffer = ScanBuff.GetBuffer(source, fallbackCodePage); this.lNum = 0; this.code = '\n'; // to initialize yyline, yycol and lineStart GetCode(); } #endif // !BYTEMODE #endif // !NOFILES // ============================================================== #if BABEL // // Get the next token for Visual Studio // // "state" is the inout mode variable that maintains scanner // state between calls, using the EolState property. In principle, // if the calls of EolState are costly set could be called once // only per line, at the start; and get called only at the end // of the line. This needs more infrastructure ... // public int GetNext(ref int state, out int start, out int end) { Tokens next; int s, e; s = state; // state at start EolState = state; next = (Tokens)Scan(); state = EolState; e = state; // state at end; start = tokPos; end = tokEPos - 1; // end is the index of last char. return (int)next; } #endif // BABEL // ======== AbstractScanner<> Implementation ========= [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex")] public override int yylex() { // parserMax is set by reflecting on the Tokens // enumeration. If maxParseToken is defined // that is used, otherwise int.MaxValue is used. int next; do { next = Scan(); } while (next >= parserMax); return next; } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int yypos { get { return tokPos; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int yyline { get { return tokLin; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] int yycol { get { return tokCol; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yytext")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yytext")] public string yytext { get { if (tokTxt == null) tokTxt = buffer.GetString(tokPos, tokEPos); return tokTxt; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void yyless(int n) { buffer.Pos = tokPos; // Must read at least one char, so set before start. cCol = tokCol - 1; GetCode(); // Now ensure that line counting is correct. lNum = tokLin; // And count the rest of the text. for (int i = 0; i < n; i++) GetCode(); MarkEnd(); } // // It would be nice to count backward in the text // but it does not seem possible to re-establish // the correct column counts except by going forward. // [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void _yytrunc(int n) { yyless(yyleng - n); } // // This is painful, but we no longer count // codepoints. For the overwhelming majority // of cases the single line code is fast, for // the others, well, at least it is all in the // buffer so no files are touched. Note that we // can't use (tokEPos - tokPos) because of the // possibility of surrogate pairs in the token. // [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyleng")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyleng")] public int yyleng { get { #if BYTEMODE return tokEPos - tokPos; #else if (tokELin == tokLin) return tokECol - tokCol; else { int ch; int count = 0; int save = buffer.Pos; buffer.Pos = tokPos; do { ch = buffer.Read(); if (!char.IsHighSurrogate((char)ch)) count++; } while (buffer.Pos < tokEPos && ch != ScanBuff.EndOfFile); buffer.Pos = save; return count; } #endif // BYTEMODE } } // ============ methods available in actions ============== [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal int YY_START { get { return currentScOrd; } set { currentScOrd = value; currentStart = startState[value]; } } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal void BEGIN(int next) { currentScOrd = next; currentStart = startState[next]; } // ============== The main tokenizer code ================= int Scan() { try { for (; ; ) { int next; // next state to enter #if BACKUP Result rslt = Result.noMatch; #endif // BACKUP #if LEFTANCHORS for (;;) { // Discard characters that do not start any pattern. // Must check the left anchor condition after *every* GetCode! state = ((cCol == 0) ? anchorState[currentScOrd] : currentStart); if ((next = NextState()) != goStart) break; // LOOP EXIT HERE... GetCode(); } #else // !LEFTANCHORS state = currentStart; while ((next = NextState()) == goStart) // At this point, the current character has no // transition from the current state. We discard // the "no-match" char. In traditional LEX such // characters are echoed to the console. GetCode(); #endif // LEFTANCHORS // At last, a valid transition ... MarkToken(); state = next; GetCode(); while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum #if BACKUP if (state <= maxAccept && next > maxAccept) // need to prepare backup data { // ctx is an object. The fields may be // mutated by the call to Recurse2. // On return the data in ctx is the // *latest* accept state that was found. rslt = Recurse2(ref ctx, next); if (rslt == Result.noMatch) RestoreStateAndPos(ref ctx); break; } else #endif // BACKUP { state = next; GetCode(); } if (state <= maxAccept) { MarkEnd(); #region ActionSwitch #pragma warning disable 162 switch (state) { case eofNum: if (yywrap()) return (int)Tokens.EOF; break; case 1: return (int)Tokens.NODIRECTIVE; break; case 2: BEGIN(COMMENT); break; case 3: BEGIN(COMMENT1); break; case 4: BEGIN(INSIDEDIRECTIVE); break; case 5: { } break; case 6: { } break; case 7: yylval = new Directive(yytext, new LexLocation(tokLin, tokCol, tokELin, tokECol)); var t1 = yylloc; return (int)Tokens.DIRECTIVE; break; case 8: BEGIN(INITIAL); break; case 9: { } break; case 10: BEGIN(INITIAL); break; case 11: case 12: { } break; case 13: BEGIN(INITIAL); break; default: break; } #pragma warning restore 162 #endregion } } } // end try finally { // User-specified epilog to scan() yylloc = new LexLocation(tokLin, tokCol, tokELin, tokECol); // End, user-specified epilog } // end finally } #if BACKUP Result Recurse2(ref Context ctx, int next) { // Assert: at entry "state" is an accept state AND // NextState(state, code) != goStart AND // NextState(state, code) is not an accept state. // SaveStateAndPos(ref ctx); state = next; GetCode(); while ((next = NextState()) > eofNum) { if (state <= maxAccept && next > maxAccept) // need to update backup data SaveStateAndPos(ref ctx); state = next; if (state == eofNum) return Result.accept; GetCode(); } return (state <= maxAccept ? Result.accept : Result.noMatch); } void SaveStateAndPos(ref Context ctx) { ctx.bPos = buffer.Pos; ctx.rPos = readPos; ctx.cCol = cCol; ctx.lNum = lNum; ctx.state = state; ctx.cChr = code; } void RestoreStateAndPos(ref Context ctx) { buffer.Pos = ctx.bPos; readPos = ctx.rPos; cCol = ctx.cCol; lNum = ctx.lNum; state = ctx.state; code = ctx.cChr; } #endif // BACKUP // ============= End of the tokenizer code ================ #if STACK [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal void yy_clear_stack() { scStack.Clear(); } [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal int yy_top_state() { return scStack.Peek(); } internal void yy_push_state(int state) { scStack.Push(currentScOrd); BEGIN(state); } internal void yy_pop_state() { // Protect against input errors that pop too far ... if (scStack.Count > 0) { int newSc = scStack.Pop(); BEGIN(newSc); } // Otherwise leave stack unchanged. } #endif // STACK [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal void ECHO() { Console.Out.Write(yytext); } #region UserCodeSection /*public override void yyerror(string format, params object[] args) { string errorMsg = parsertools.CreateErrorString(yytext,args); parsertools.AddError(errorMsg,new LexLocation(tokLin, tokCol, tokELin, tokECol)); }*/ #endregion } // end class $Scanner // ============================================================== // // This code automatically produced from an embedded resource. // Do not edit this file, or it will become incompatible with // the specification from which it was generated. // // ============================================================== // Code copied from GPLEX embedded resource [Serializable] public class BufferException : Exception { public BufferException() { } public BufferException(string message) : base(message) { } public BufferException(string message, Exception innerException) : base(message, innerException) { } protected BufferException(SerializationInfo info, StreamingContext context) : base(info, context) { } } public abstract class ScanBuff { private string fileNm; public const int EndOfFile = -1; public const int UnicodeReplacementChar = 0xFFFD; public bool IsFile { get { return (fileNm != null); } } public string FileName { get { return fileNm; } set { fileNm = value; } } public abstract int Pos { get; set; } public abstract int Read(); public virtual void Mark() { } public abstract string GetString(int begin, int limit); public static ScanBuff GetBuffer(string source) { return new StringBuffer(source); } public static ScanBuff GetBuffer(IList source) { return new LineBuffer(source); } public static ScanBuff GetBuffer(Stream source) { return new BuildBuffer(source); } #if (!BYTEMODE) public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) { return new BuildBuffer(source, fallbackCodePage); } #endif } #region Buffer classes // ============================================================== // ===== Definitions for various ScanBuff derived classes ==== // ============================================================== // =============== String input ================ // ============================================================== /// /// This class reads characters from a single string as /// required, for example, by Visual Studio language services /// sealed class StringBuffer : ScanBuff { string str; // input buffer int bPos; // current position in buffer int sLen; public StringBuffer(string source) { this.str = source; this.sLen = source.Length; this.FileName = null; } public override int Read() { if (bPos < sLen) return str[bPos++]; else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line else { bPos++; return EndOfFile; } // two strikes and you're out! } public override string GetString(int begin, int limit) { // "limit" can be greater than sLen with the BABEL // option set. Read returns a "virtual" EOL if // an attempt is made to read past the end of the // string buffer. Without the guard any attempt // to fetch yytext for a token that includes the // EOL will throw an index exception. if (limit > sLen) limit = sLen; if (limit <= begin) return ""; else return str.Substring(begin, limit - begin); } public override int Pos { get { return bPos; } set { bPos = value; } } public override string ToString() { return "StringBuffer"; } } // ============================================================== // The LineBuff class contributed by Nigel Horspool, // nigelh@cs.uvic.cs // ============================================================== sealed class LineBuffer : ScanBuff { IList line; // list of source lines from a file int numLines; // number of strings in line list string curLine; // current line in that list int cLine; // index of current line in the list int curLen; // length of current line int curLineStart; // position of line start in whole file int curLineEnd; // position of line end in whole file int maxPos; // max position ever visited in whole file int cPos; // ordinal number of code in source // Constructed from a list of strings, one per source line. // The lines have had trailing '\n' characters removed. public LineBuffer(IList lineList) { line = lineList; numLines = line.Count; cPos = curLineStart = 0; curLine = (numLines > 0 ? line[0] : ""); maxPos = curLineEnd = curLen = curLine.Length; cLine = 1; FileName = null; } public override int Read() { if (cPos < curLineEnd) return curLine[cPos++ - curLineStart]; if (cPos++ == curLineEnd) return '\n'; if (cLine >= numLines) return EndOfFile; curLine = line[cLine]; curLen = curLine.Length; curLineStart = curLineEnd + 1; curLineEnd = curLineStart + curLen; if (curLineEnd > maxPos) maxPos = curLineEnd; cLine++; return curLen > 0 ? curLine[0] : '\n'; } // To speed up searches for the line containing a position private int cachedPosition; private int cachedIxdex; private int cachedLineStart; // Given a position pos within the entire source, the results are // ix -- the index of the containing line // lstart -- the position of the first character on that line private void findIndex(int pos, out int ix, out int lstart) { if (pos >= cachedPosition) { ix = cachedIxdex; lstart = cachedLineStart; } else { ix = lstart = 0; } for (; ; ) { int len = line[ix].Length + 1; if (pos < lstart + len) break; lstart += len; ix++; } cachedPosition = pos; cachedIxdex = ix; cachedLineStart = lstart; } public override string GetString(int begin, int limit) { if (begin >= maxPos || limit <= begin) return ""; int endIx, begIx, endLineStart, begLineStart; findIndex(begin, out begIx, out begLineStart); int begCol = begin - begLineStart; findIndex(limit, out endIx, out endLineStart); int endCol = limit - endLineStart; string s = line[begIx]; if (begIx == endIx) { // the usual case, substring all on one line return (endCol <= s.Length) ? s.Substring(begCol, endCol - begCol) : s.Substring(begCol) + "\n"; } // the string spans multiple lines, yuk! StringBuilder sb = new StringBuilder(); if (begCol < s.Length) sb.Append(s.Substring(begCol)); for (; ; ) { sb.Append("\n"); s = line[++begIx]; if (begIx >= endIx) break; sb.Append(s); } if (endCol <= s.Length) { sb.Append(s.Substring(0, endCol)); } else { sb.Append(s); sb.Append("\n"); } return sb.ToString(); } public override int Pos { get { return cPos; } set { cPos = value; findIndex(cPos, out cLine, out curLineStart); curLine = line[cLine]; curLineEnd = curLineStart + curLine.Length; } } public override string ToString() { return "LineBuffer"; } } // ============================================================== // ===== class BuildBuff : for unicode text files ======== // ============================================================== class BuildBuffer : ScanBuff { // Double buffer for char stream. class BufferElement { StringBuilder bldr = new StringBuilder(); StringBuilder next = new StringBuilder(); int minIx; int maxIx; int brkIx; bool appendToNext; internal BufferElement() { } internal int MaxIndex { get { return maxIx; } } // internal int MinIndex { get { return minIx; } } internal char this[int index] { get { if (index < minIx || index >= maxIx) throw new BufferException("Index was outside data buffer"); else if (index < brkIx) return bldr[index - minIx]; else return next[index - brkIx]; } } internal void Append(char[] block, int count) { maxIx += count; if (appendToNext) this.next.Append(block, 0, count); else { this.bldr.Append(block, 0, count); brkIx = maxIx; appendToNext = true; } } internal string GetString(int start, int limit) { if (limit <= start) return ""; if (start >= minIx && limit <= maxIx) if (limit < brkIx) // String entirely in bldr builder return bldr.ToString(start - minIx, limit - start); else if (start >= brkIx) // String entirely in next builder return next.ToString(start - brkIx, limit - start); else // Must do a string-concatenation return bldr.ToString(start - minIx, brkIx - start) + next.ToString(0, limit - brkIx); else throw new BufferException("String was outside data buffer"); } internal void Mark(int limit) { if (limit > brkIx + 16) // Rotate blocks { StringBuilder temp = bldr; bldr = next; next = temp; next.Length = 0; minIx = brkIx; brkIx = maxIx; } } } BufferElement data = new BufferElement(); int bPos; // Postion index in the StringBuilder BlockReader NextBlk; // Delegate that serves char-arrays; private string EncodingName { get { StreamReader rdr = NextBlk.Target as StreamReader; return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); } } public BuildBuffer(Stream stream) { FileStream fStrm = (stream as FileStream); if (fStrm != null) FileName = fStrm.Name; NextBlk = BlockReaderFactory.Raw(stream); } #if (!BYTEMODE) public BuildBuffer(Stream stream, int fallbackCodePage) { FileStream fStrm = (stream as FileStream); if (fStrm != null) FileName = fStrm.Name; NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); } #endif /// /// Marks a conservative lower bound for the buffer, /// allowing space to be reclaimed. If an application /// needs to call GetString at arbitrary past locations /// in the input stream, Mark() is not called. /// public override void Mark() { data.Mark(bPos - 2); } public override int Pos { get { return bPos; } set { bPos = value; } } /// /// Read returns the ordinal number of the next char, or /// EOF (-1) for an end of stream. Note that the next /// code point may require *two* calls of Read(). /// /// public override int Read() { // // Characters at positions // [data.offset, data.offset + data.bldr.Length) // are available in data.bldr. // if (bPos < data.MaxIndex) { // ch0 cannot be EOF return (int)data[bPos++]; } else // Read from underlying stream { // Experimental code, blocks of page size char[] chrs = new char[4096]; int count = NextBlk(chrs, 0, 4096); if (count == 0) return EndOfFile; else { data.Append(chrs, count); return (int)data[bPos++]; } } } public override string GetString(int begin, int limit) { return data.GetString(begin, limit); } public override string ToString() { return "StringBuilder buffer, encoding: " + this.EncodingName; } } // =============== End ScanBuff-derived classes ================== public delegate int BlockReader(char[] block, int index, int number); // A delegate factory, serving up a delegate that // reads a block of characters from the underlying // encoded stream, via a StreamReader object. // public static class BlockReaderFactory { public static BlockReader Raw(Stream stream) { return delegate(char[] block, int index, int number) { byte[] b = new byte[number]; int count = stream.Read(b, 0, number); int i = 0; int j = index; for (; i < count; i++, j++) block[j] = (char)b[i]; return count; }; } #if (!BYTEMODE) public static BlockReader Get(Stream stream, int fallbackCodePage) { Encoding encoding; int preamble = Preamble(stream); if (preamble != 0) // There is a valid BOM here! encoding = Encoding.GetEncoding(preamble); else if (fallbackCodePage == -1) // Fallback is "raw" bytes return Raw(stream); else if (fallbackCodePage != -2) // Anything but "guess" encoding = Encoding.GetEncoding(fallbackCodePage); else // This is the "guess" option { int guess = new Guesser(stream).GuessCodePage(); stream.Seek(0, SeekOrigin.Begin); if (guess == -1) // ==> this is a 7-bit file encoding = Encoding.ASCII; else if (guess == 65001) encoding = Encoding.UTF8; else // ==> use the machine default encoding = Encoding.Default; } StreamReader reader = new StreamReader(stream, encoding); return reader.Read; } static int Preamble(Stream stream) { int b0 = stream.ReadByte(); int b1 = stream.ReadByte(); if (b0 == 0xfe && b1 == 0xff) return 1201; // UTF16BE if (b0 == 0xff && b1 == 0xfe) return 1200; // UTF16LE int b2 = stream.ReadByte(); if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) return 65001; // UTF8 // // There is no unicode preamble, so we // return denoter for the machine default. // stream.Seek(0, SeekOrigin.Begin); return 0; } #endif // !BYTEMODE } #endregion Buffer classes // ============================================================== // ============ class CodePageHandling ============= // ============================================================== public static class CodePageHandling { public static int GetCodePage(string option) { string command = option.ToUpperInvariant(); if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) command = command.Substring(9); try { if (command.Equals("RAW")) return -1; else if (command.Equals("GUESS")) return -2; else if (command.Equals("DEFAULT")) return 0; else if (char.IsDigit(command[0])) return int.Parse(command, CultureInfo.InvariantCulture); else { Encoding enc = Encoding.GetEncoding(command); return enc.CodePage; } } catch (FormatException) { Console.Error.WriteLine( "Invalid format \"{0}\", using machine default", option); } catch (ArgumentException) { Console.Error.WriteLine( "Unknown code page \"{0}\", using machine default", option); } return 0; } } #region guesser #if (!BYTEMODE) // ============================================================== // ============ Encoding Guesser ============= // ============================================================== /// /// This class provides a simple finite state automaton that /// scans the file looking for (1) valid UTF-8 byte patterns, /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. /// The method then guesses whether it is UTF-8 or maybe some /// local machine default encoding. This works well for the /// various Latin encodings. /// internal class Guesser { ScanBuff buffer; public int GuessCodePage() { return Scan(); } const int maxAccept = 10; const int initial = 0; const int eofNum = 0; const int goStart = -1; const int INITIAL = 0; const int EndToken = 0; #region user code /* * Reads the bytes of a file to determine if it is * UTF-8 or a single-byte code page file. */ public long utfX; public long uppr; #endregion user code int state; int currentStart = startState[0]; int code; #region ScannerTables static int[] startState = new int[] { 11, 0 }; #region CharacterMap static sbyte[] map = new sbyte[256] { /* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; #endregion static sbyte[][] nextState = new sbyte[][] { new sbyte[] {0, 0, 0, 0, 0, 0}, new sbyte[] {-1, -1, 10, -1, -1, -1}, new sbyte[] {-1, -1, -1, -1, -1, -1}, new sbyte[] {-1, -1, 8, -1, -1, -1}, new sbyte[] {-1, -1, 5, -1, -1, -1}, new sbyte[] {-1, -1, 6, -1, -1, -1}, new sbyte[] {-1, -1, 7, -1, -1, -1}, null, new sbyte[] {-1, -1, 9, -1, -1, -1}, null, null, new sbyte[] {-1, 1, 2, 3, 4, 2} }; [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] // Reason for suppression: cannot have self-reference in array initializer. static Guesser() { nextState[7] = nextState[2]; nextState[9] = nextState[2]; nextState[10] = nextState[2]; } int NextState() { if (code == ScanBuff.EndOfFile) return eofNum; else return nextState[state][map[code]]; } #endregion public Guesser(System.IO.Stream file) { SetSource(file); } public void SetSource(System.IO.Stream source) { this.buffer = new BuildBuffer(source); code = buffer.Read(); } int Scan() { for (; ; ) { int next; state = currentStart; while ((next = NextState()) == goStart) code = buffer.Read(); state = next; code = buffer.Read(); while ((next = NextState()) > eofNum) { state = next; code = buffer.Read(); } if (state <= maxAccept) { #region ActionSwitch #pragma warning disable 162 switch (state) { case eofNum: switch (currentStart) { case 11: if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ else if (uppr * 10 > utfX) return 0; /* default code page */ else return 65001; /* UTF-8 encoding */ break; } return EndToken; case 1: // Recognized '{Upper128}', Shortest string "\xC0" case 2: // Recognized '{Upper128}', Shortest string "\x80" case 3: // Recognized '{Upper128}', Shortest string "\xE0" case 4: // Recognized '{Upper128}', Shortest string "\xF0" uppr++; break; case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" uppr += 2; break; case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" uppr += 3; break; case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" utfX += 3; break; case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" uppr += 2; break; case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" utfX += 2; break; case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" utfX++; break; default: break; } #pragma warning restore 162 #endregion } } } } // end class Guesser #endif // !BYTEMODE #endregion // End of code copied from embedded resource } // end namespace