# TextLexer.java

    // $ANTLR 2.7.7 (20060906): "text.g" -> "TextLexer.java"$

    /*
    ** Module   : TextLexer.java
    **            TextParser.java
    **            PreprocTokenTypes.java
    ** Abstract : This grammar is the top level recognizer of the Progress
    **            language preprocessor input. It recognizes tabs, nested
    **            comments, single and double quoted multiline strings and
    **            the code.
    **
    ** Copyright (c) 2004-2019, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- -T- --JPRM-- ----------------Description-----------------
    ** 001 GES 20041116 ADD   @18832 WARNING, THIS IS A GENERATED FILE!!!
    **                               DO NOT EDIT THIS FILE. The original source
    **                               file is text.g!
    */

    package com.goldencode.p2j.preproc;

    import java.io.*;
    import java.util.*;
    import java.util.logging.*;
    import antlr.collections.AST;
    import antlr.collections.impl.*;
    import antlr.debug.misc.*;
    import antlr.*;
    import com.goldencode.util.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.util.*;

    import java.io.InputStream;
    import antlr.TokenStreamException;
    import antlr.TokenStreamIOException;
    import antlr.TokenStreamRecognitionException;
    import antlr.CharStreamException;
    import antlr.CharStreamIOException;
    import antlr.ANTLRException;
    import java.io.Reader;
    import java.util.Hashtable;
    import antlr.CharScanner;
    import antlr.InputBuffer;
    import antlr.ByteBuffer;
    import antlr.CharBuffer;
    import antlr.Token;
    import antlr.CommonToken;
    import antlr.RecognitionException;
    import antlr.NoViableAltForCharException;
    import antlr.MismatchedCharException;
    import antlr.TokenStream;
    import antlr.ANTLRHashString;
    import antlr.LexerSharedInputState;
    import antlr.collections.impl.BitSet;
    import antlr.SemanticException;

    /**
     * Tokenizes the input stream of characters from the Progress source
     * file and returns tokens to the caller according to the needs of
     * preprocessor.
     * <p>
     * The lexer recognizes the current context such as comments, strings
     * or code and keeps track of the context switches using
     * <code>{@link Environment#setInComment(boolean)}</code> and
     * <code>{@link Environment#setInString(boolean)}</code> methods.
     *
     * @see  TextParser
     * @see  Environment
     */
    public class TextLexer extends antlr.CharScanner implements PreprocTokenTypes, TokenStream
     {

       /** Logger. */
       private static final ConversionStatus LOG = ConversionStatus.get(TextLexer.class);

       /** keeps the reference to the shared environment */
       private Environment env = null;

       /** Nesting level for comments. */
       private int commentNesting = 0;

       /** Control matched quote processing in strings. */
       private boolean brokenStringMatching = false;

       /**
        * Constructor. Creates a lexer attached to the input stream
        * taken from the environment. Saves the environment
        * for future needs.
        *
        * @param    env
        *           Shared preprocessor environment.
        */
       public TextLexer(Environment env)
       {
          this(new CharBuffer(env.getIns()));
          this.env = env;
          this.env.setLsi(this.getInputState());
       }

       /**
         * Expands tabs with spaces right in the ANTLRStringBuilder text.
         * This version is used here because according to experiments,
         * the tabs expansion for the preprocessor variables is immediate.
         */
       public void tab()
       {
          int c  = getColumn();
          int nc = (((c-1)/tabsize) + 1) * tabsize + 1;  // calculate tab stop
          setColumn(nc);

          text.setLength(text.length() - 1);    // replace '\t' in the buffer
          for (; c < nc; c ++)                  // add more spaces
             text.append(' ');
       }

       /**
        * Tests the token text against the literals table and provides
        * correct token types for the abbreviated preprocessor statement
        * keywords.
        */
       public int testLiteralsTable(int ttype)
       {
          // support abbreviations in some of the possible literals
          String[] abbrs =
          {
             "&global-define",
             "&scoped-define",
             "&undefine",
             "&analyze-suspend",
             "&analyze-resume"
          };

          // minimum lengths of each of the above possible abbreviations
          int[] minsz = {5, 5, 6, 10, 10};

          String  itoken = new String(text.getBuffer(), 0, text.length());
          String  token  = itoken.toLowerCase();
          int     ltoken = token.length();

          for (int i = 0; i < abbrs.length; i ++)
          {
             if (abbrs[i].startsWith(token) && ltoken >= minsz[i])
             {
                token = abbrs[i];
             }
          }

          hashString.setString(token);

          Integer literalsIndex = (Integer)literals.get(hashString);

          if (literalsIndex != null)
          {
             ttype = literalsIndex.intValue();
          }

          // disable normal quote matching in STRING until the NL is reached
          // (this is necessary since unmatched quotes are allowed in defines
          // and all downstream parsing is really broken otherwise)
          if (ttype == AGLOBAL || ttype == ASCOPED || ttype == AMESSAGE)
          {
             brokenStringMatching = true;
          }

          return ttype;
       }

       /**
        * Display a debug message along with a representation of LA(1).
        *
        * @param    msg
        *           A debug message.
        */
       public void debug(String msg)
       throws CharStreamException
       {
          LOG.log(Level.FINE, String.format("%s '%c' (0x%04X)\n", msg, LA(1), (short) LA(1)));
       }
    public TextLexer(InputStream in) {
    	this(new ByteBuffer(in));
    }
    public TextLexer(Reader in) {
    	this(new CharBuffer(in));
    }
    public TextLexer(InputBuffer ib) {
    	this(new LexerSharedInputState(ib));
    }
    public TextLexer(LexerSharedInputState state) {
    	super(state);
    	caseSensitiveLiterals = false;
    	setCaseSensitive(false);
    	literals = new Hashtable();
    	literals.put(new ANTLRHashString("&endif", this), Integer.valueOf(24));
    	literals.put(new ANTLRHashString("&analyze-suspend", this), Integer.valueOf(25));
    	literals.put(new ANTLRHashString("&message", this), Integer.valueOf(18));
    	literals.put(new ANTLRHashString("&if", this), Integer.valueOf(20));
    	literals.put(new ANTLRHashString("&scoped-define", this), Integer.valueOf(17));
    	literals.put(new ANTLRHashString("&global-define", this), Integer.valueOf(16));
    	literals.put(new ANTLRHashString("&elseif", this), Integer.valueOf(22));
    	literals.put(new ANTLRHashString("&then", this), Integer.valueOf(21));
    	literals.put(new ANTLRHashString("&undefine", this), Integer.valueOf(19));
    	literals.put(new ANTLRHashString("&else", this), Integer.valueOf(23));
    	literals.put(new ANTLRHashString("&analyze-resume", this), Integer.valueOf(26));
    }

    public Token nextToken() throws TokenStreamException {
    	Token theRetToken=null;
    tryAgain:
    	for (;;) {
    		Token _token = null;
    		int _ttype = Token.INVALID_TYPE;
    		resetText();
    		try {   // for char stream error handling
    			try {   // for lexical error handling
    				switch ( LA(1)) {
    				case '\t':  case ' ':
    				{
    					mWS(true);
    					theRetToken=_returnToken;
    					break;
    				}
    				case '\n':
    				{
    					mNL(true);
    					theRetToken=_returnToken;
    					break;
    				}
    				case '"':  case '\'':
    				{
    					mSTRING(true);
    					theRetToken=_returnToken;
    					break;
    				}
    				case '&':
    				{
    					mASTMT(true);
    					theRetToken=_returnToken;
    					break;
    				}
    				case '(':  case ')':  case ',':  case '<':
    				case '=':  case '>':  case '[':  case ']':
    				case '{':  case '|':  case '}':
    				{
    					mBREAK_CODE_CHUNKS(true);
    					theRetToken=_returnToken;
    					break;
    				}
    				default:
    					if ((_tokenSet_0.member(LA(1)))) {
    						mCODE(true);
    						theRetToken=_returnToken;
    					}
    				else {
    					if (LA(1)==EOF_CHAR) {uponEOF(); _returnToken = makeToken(Token.EOF_TYPE);}
    				else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());}
    				}
    				}
    				if ( _returnToken==null ) continue tryAgain; // found SKIP token
    				_ttype = _returnToken.getType();
    				_ttype = testLiteralsTable(_ttype);
    				_returnToken.setType(_ttype);
    				return _returnToken;
    			}
    			catch (RecognitionException e) {
    				throw new TokenStreamRecognitionException(e);
    			}
    		}
    		catch (CharStreamException cse) {
    			if ( cse instanceof CharStreamIOException ) {
    				throw new TokenStreamIOException(((CharStreamIOException)cse).io);
    			}
    			else {
    				throw new TokenStreamException(cse.getMessage());
    			}
    		}
    	}
    }

    /**
     * Matches any amount of continguous whitespace (spaces and tabs) in a program.
     * Newlines are NOT matched.
     * <p>
     * This is a  top level lexer rule which means that there is an associated
     * <code>WS</code> token.
     */
    	public final void mWS(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = WS;
    		int _saveIndex;

    		{
    		int _cnt55=0;
    		_loop55:
    		do {
    			switch ( LA(1)) {
    			case ' ':
    			{
    				match(' ');
    				break;
    			}
    			case '\t':
    			{
    				match('\t');
    				break;
    			}
    			default:
    			{
    				if ( _cnt55>=1 ) { break _loop55; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());}
    			}
    			}
    			_cnt55++;
    		} while (true);
    		}
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches a single newline character.
     * <p>
     * This is a top level lexer rule which means that there is an associated
     * <code>NL</code> token.
     */
    	public final void mNL(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = NL;
    		int _saveIndex;

    		match('\n');

    		newline();

    		// if a scoped or global define has ended, re-enable STRING processing
    		if (brokenStringMatching)
    		{
    		brokenStringMatching = false;
    		}

    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches Progress language strings. Brackets the string with a pair
     * of calls to <code>{@link Environment#setInString(boolean)}</code>
     * so the preprocessor always knows the context.
     * <p>
     * Depending on -keeptildes preprocessor options, newlines may be required
     * to be deleted from the strings. This requirement does not extend to the
     * characters which are coded as escape sequences, though. The final action in
     * this rule postprocesses the string contents so they comply with the
     * requirement.
     * <p>
     * To tell the original newlines from the escaped ones, ClearStream uses the
     * following technique. For every converted character, that becomes newline,
     * an escape character is inserted in front of it into the stream. The
     * postprocessing action is supposed to detect this escape character, discard
     * it but leave the newline that follows untouched. The original newlines come
     * without escape characters and are deleted.
     * <p>
     * For this technique to work, an escape character has to be assigned which
     * normally can't be seen in the stream. Such a character is CR, due to the
     * fact that ClearStream strips off all original CRs and replaces them with
     * NLs. But the same problem of converted CRs still exists. Fortunately, it is
     * solved in a similar way. As the result, the escape character, CR, is
     * inserted in front of every converted CR or NL. The valid escape sequences
     * that ClearStream produces are CR CR and CR NL. In both cases, the first CR
     * has to be removed from the stream and the second character left untouched.
     * <p>
     * One more complication is due to the fact that there is no on time state
     * change signalling between the lexer and the ClearStream code. Although this
     * rule signals the end of string by calling <code>inString(false)</code>, the
     * call is one character late because of the k=2 lookahead in the lexer. Late
     * signalling leaves a hole in the algorithm for a wrong escape character
     * insertion just outside the string, which should not happen. Consider the
     * following stream:
     * <pre>
     *    "..."~n~n
     * </pre>
     * Due to the late signalling, ClearStream reads and converts the ~n Progress
     * escape sequence before this rule signals the end of string. As the result,
     * ClearStream produces the following stream:
     * <pre>
     *    "..." CR NL NL
     * </pre>
     * Notice that only the first NL gets escaped with CR, the second NL comes
     * when the end of string signal has been received. This issue is fixed by
     * checking the next character in the lookahead buffer LA(1) in the
     * postprocessing rule. If it is CR, it has to be discarded, as it only can be
     * there as the result of this late signalling.
     * <p>
     * This is a top level lexer rule which means that there is an associated
     * <code>STRING</code> token.
     * <p>
     * If there is an escaped <code>null</code> character in the string literal,
     * it must be "space-ified" by converting it and all following characters
     * into spaces (including the proper reduction of other following escape
     * sequences into a single space character).  One can encode a
     * <code>null</code> escape sequence in a string literal in Progress BUT a
     * <code>null</code> character will NEVER end up in the result!  This method
     * duplicates that processing.
     *
     * @throws   antlr.RecognitionException
     * @throws   antlr.CharStreamException
     * @throws   antlr.TokenStreamException
     */
    	public final void mSTRING(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = STRING;
    		int _saveIndex;

    		if (!brokenStringMatching)
    		{
    		env.setInString(true);
    		}

    		{
    		switch ( LA(1)) {
    		case '\'':
    		{
    			mASTRING(false);
    			break;
    		}
    		case '"':
    		{
    			mQSTRING(false);
    			break;
    		}
    		default:
    		{
    			throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
    		}
    		}
    		}

    		if (!brokenStringMatching)
    		{
    		env.setInString(false);

    		text.setLength(Preprocessor.stripNL(text.getBuffer(),
    		text.length(),
    		env.getOpt().isKeepTildes()));
    		if (LA(1) == 0x0D)
    		{
    		consume();
    		text.setLength(text.length() - 1);
    		}

    		boolean unixEsc = env.getOpt().isUnixEscapes();

    		// implement spacifying transformation here
    		setText(character.progressSpacifyNull(getText(), unixEsc));
    		}

    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches an opening single quote, arbitrary contents and an ending single
     * quote.  Two continguous single quote characters and an escape prefixed
     * single quote character are accepted as contents (they do not terminate
     * the string).
     * <p>
     * Any newlines inside the string are identified and the lexer's internal
     * newline counter is properly maintained.  Note that all such newlines
     * are maintained in the output string because these are the escaped chars
     * that have been left behind by the preprocessor.  If such characters are not
     * escaped, then the preprocessor removes such chars (carriage returns and
     * line feeds).  This is how Progress 4GL handles these characters.  This
     * means that in a raw source file that has not been preprocessed, a string
     * literal can be split across any number of lines and the Progress
     * preprocessor will put the string back together, ignoring the carriage
     * returns and newlines.
     * <p>
     * Tilde and the backslash can BOTH be Progress escape characters and as such,
     * they need special attention. Backslash is only honored if UNIX escapes
     * mode is set on. In particular, this rule must separately match 3 or 6
     * constructs as part of a string depending on which escape sequences are
     * honored.
     * <p>
     * <pre>
     *    ~'
     *    \'
     *    ~~
     *    ~\
     *    \\
     *    \~
     *    ~
     *    \
     * </pre>
     * <p>
     * The first 2 are a way of embedding a quote in a string.  The next 4 are
     * important because if this rule were to encounter '~~', '\\', '~\' or '\~'
     * (which are valid strings) the rule would consume the first escape and then
     * encounter an escaped quote which would not terminate the string properly,
     * leading to a non-ending string.  So this rule matches on any escaped escape
     * char to eliminate this situation.  Finally, a single ~ or \ that is not
     * followed by a &quot; or a duplicate escape char is matched as a single
     * character.  This is required (one may think that the closure rule
     * should handle this case) because when ANTLR sees a ~ or \ in the leftmost
     * position of the alternatives, it DOES NOT include it in the list of
     * 'everything that is not a closing quote'.  Likewise \n is not included.
     * Of course, these tilde constructions must be placed in a specific order
     * and if so, the ambiguity warnings that ANTLR reports can be disabled.
     * <p>
     * The greedy option does not need to be disabled here as the closure rule
     * termination is built into the subrule itself: it accepts anything that
     * isn't an unescaped single quote character (see above).
     * <p>
     * Tabs and spaces are maintained inside strings.
     */
    	protected final void mASTRING(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = ASTRING;
    		int _saveIndex;

    		boolean honor = env.getOpt().isUnixEscapes();

    		{
    		match('\'');
    		{
    		_loop85:
    		do {
    			if ((LA(1)=='~') && (LA(2)=='\'')) {
    				match("~\'");
    			}
    			else if ((LA(1)=='~') && (LA(2)=='~')) {
    				match("~~");
    			}
    			else if ((LA(1)=='~') && (LA(2)=='\\')) {
    				match("~\\");
    			}
    			else if (((LA(1)=='\\') && (LA(2)=='\''))&&( honor )) {
    				match("\\\'");
    			}
    			else if (((LA(1)=='\\') && (LA(2)=='\\'))&&( honor )) {
    				match("\\\\");
    			}
    			else if (((LA(1)=='\\') && (LA(2)=='~'))&&( honor )) {
    				match("\\~");
    			}
    			else if ((LA(1)=='\'') && (LA(2)=='\'')) {
    				match("\'\'");
    			}
    			else if ((LA(1)=='~') && (true)) {
    				match('~');
    			}
    			else if ((LA(1)=='\\') && (true)) {
    				match('\\');
    			}
    			else if (((LA(1)=='\n'))&&( !brokenStringMatching )) {
    				match('\n');
    				newline();
    			}
    			else if ((_tokenSet_1.member(LA(1)))) {
    				matchNot('\'');
    			}
    			else {
    				break _loop85;
    			}

    		} while (true);
    		}
    		{
    		if (((LA(1)=='\'') && (true))&&( !brokenStringMatching )) {
    			match('\'');
    		}
    		else if (( true )&&( brokenStringMatching )) {
    			{
    			if ((LA(1)=='\'')) {
    				match('\'');
    			}
    			else {
    			}

    			}
    		}
    		else {
    			throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
    		}

    		}
    		}
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches an opening double quote, arbitrary contents and an ending double
     * quote.  Two continguous double quote characters and an escape prefixed
     * double quote character are accepted as contents (they do not terminate
     * the string).
     * <p>
     * Any newlines inside the string are identified and the lexer's internal
     * newline counter is properly maintained.  Note that all such newlines
     * are maintained in the output string because these are the escaped chars
     * that have been left behind by the preprocessor.  If such characters are not
     * escaped, then the preprocessor removes such chars (carriage returns and
     * line feeds).  This is how Progress 4GL handles these characters.  This
     * means that in a raw source file that has not been preprocessed, a string
     * literal can be split across any number of lines and the Progress
     * preprocessor will put the string back together, ignoring the carriage
     * returns and newlines.
     * <p>
     * Tilde and the backslash can BOTH be Progress escape characters and as such,
     * they need special attention. Backslash is only honored if UNIX escapes
     * mode is set on. In particular, this rule must separately match 3 or 6
     * constructs as part of a string depending on which escape sequences are
     * honored.
     * <p>
     * <pre>
     *    ~&quot;
     *    \&quot;
     *    ~~
     *    ~\
     *    \\
     *    \~
     *    ~
     *    \
     * </pre>
     * <p>
     * The first 2 are a way of embedding a quote in a string.  The next 4 are
     * important because if this rule were to encounter '~~', '\\', '~\' or '\~'
     * (which are valid strings) the rule would consume the first escape and then
     * encounter an escaped quote which would not terminate the string properly,
     * leading to a non-ending string.  So this rule matches on any escaped escape
     * char to eliminate this situation.  Finally, a double ~ or \ that is not
     * followed by a &quot; or a duplicate escape char is matched as a double
     * character.  This is required (one may think that the closure rule
     * should handle this case) because when ANTLR sees a ~ or \ in the leftmost
     * position of the alternatives, it DOES NOT include it in the list of
     * 'everything that is not a closing quote'.  Likewise \n is not included.
     * Of course, these tilde constructions must be placed in a specific order
     * and if so, the ambiguity warnings that ANTLR reports can be disabled.
     * <p>
     * The greedy option does not need to be disabled here as the closure rule
     * termination is built into the subrule itself: it accepts anything that
     * isn't an unescaped double quote character (see above).
     * <p>
     * Tabs and spaces are maintained inside strings.
     */
    	protected final void mQSTRING(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = QSTRING;
    		int _saveIndex;

    		boolean honor = env.getOpt().isUnixEscapes();

    		{
    		match('\"');
    		{
    		_loop91:
    		do {
    			if ((LA(1)=='~') && (LA(2)=='"')) {
    				match("~\"");
    			}
    			else if ((LA(1)=='~') && (LA(2)=='~')) {
    				match("~~");
    			}
    			else if ((LA(1)=='~') && (LA(2)=='\\')) {
    				match("~\\");
    			}
    			else if (((LA(1)=='\\') && (LA(2)=='"'))&&( honor )) {
    				match("\\\"");
    			}
    			else if (((LA(1)=='\\') && (LA(2)=='\\'))&&( honor )) {
    				match("\\\\");
    			}
    			else if (((LA(1)=='\\') && (LA(2)=='~'))&&( honor )) {
    				match("\\~");
    			}
    			else if ((LA(1)=='"') && (LA(2)=='"')) {
    				match("\"\"");
    			}
    			else if ((LA(1)=='~') && (true)) {
    				match('~');
    			}
    			else if ((LA(1)=='\\') && (true)) {
    				match('\\');
    			}
    			else if (((LA(1)=='\n'))&&( !brokenStringMatching )) {
    				match('\n');
    				newline();
    			}
    			else if ((_tokenSet_2.member(LA(1)))) {
    				matchNot('\"');
    			}
    			else {
    				break _loop91;
    			}

    		} while (true);
    		}
    		{
    		if (((LA(1)=='"') && (true))&&( !brokenStringMatching )) {
    			match('\"');
    		}
    		else if (( true )&&( brokenStringMatching )) {
    			{
    			if ((LA(1)=='"')) {
    				match('\"');
    			}
    			else {
    			}

    			}
    		}
    		else {
    			throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
    		}

    		}
    		}
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches any ampersand prefaced symbolic name including all recognized or
     * unrecognized preprocessor directives. Such a statement starts with an
     * <code>'&amp;'</code> followed by letters and possibly the hyphen character.
     * <p>
     * The resulting token's text will be compared with the literals table by
     * the <code>testLiteralsTable</code> method of this class. The matching is
     * done case-insensitively and a subset of the symbols can be matched with
     * an abbreviated form. The token type replacement occurs as follows:
     * <p>
     * <pre>
     * Token Type   Matched Text        Minimum Abbreviation Chars
     * -----------  ------------------  --------------------------
     * AGLOBAL      &amp;global-define      5
     * ASCOPED      &amp;scoped-define      5
     * AMESSAGE     &amp;message            n/a
     * AUNDEFINE    &amp;undefine           6
     * AIF          &amp;if                 n/a
     * ATHEN        &amp;then               n/a
     * AELSEIF      &amp;elseif             n/a
     * AELSE        &amp;else               n/a
     * AENDIF       &amp;endif              n/a
     * ASUSPEND     &amp;analyze-suspend    n/a
     * ARESUME      &amp;analyze-resume     n/a
     * </pre>
     * <p>
     * This is a top level lexer rule which means that in the case where the
     * statement is unrecognized, there will be an associated <code>ASTMT</code>
     * token.
     * <p>
     * If the leading ampersand is not followed by a recognized statement, then
     * the ampersand and any following alphabetic characters will be returned as a
     * <code>CODE</code> token.  It is also possible to return the ampersand by
     * itself as a <code>CODE</code> token, if there are no following alphabetic
     * characters. Both of these combinations are possible in certain 4GL parsing
     * cases like shell commands the ampersand character is perfectly valid. Another
     * example is that ampersands can appear inside a variable name like my-p&amp;l-var.
     * The 4GL is just insane.
     *
     * @throws   antlr.RecognitionException
     * @throws   antlr.CharStreamException
     * @throws   antlr.TokenStreamException
     */
    	public final void mASTMT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = ASTMT;
    		int _saveIndex;

    		// we used to set this after we matched the ampersand, but it turns out that there can be
    		// constructs like &{&METADEFINE}-define where {&METADEFINE} expands to global or scoped
    		// for which the 4GL happily treats the cumulative result as &global-define or
    		// &scoped-define, this was broken when in-stmt was set too late because the next
    		// characters read were marker chars not the expanded text; by setting this earlier
    		// we eliminate the marker chars and everything works as in the 4GL
    		env.setInStatement(true);

    		match('&');
    		{
    		if ((_tokenSet_3.member(LA(1)))) {
    			{
    			int _cnt62=0;
    			_loop62:
    			do {
    				switch ( LA(1)) {
    				case 'a':  case 'b':  case 'c':  case 'd':
    				case 'e':  case 'f':  case 'g':  case 'h':
    				case 'i':  case 'j':  case 'k':  case 'l':
    				case 'm':  case 'n':  case 'o':  case 'p':
    				case 'q':  case 'r':  case 's':  case 't':
    				case 'u':  case 'v':  case 'w':  case 'x':
    				case 'y':  case 'z':
    				{
    					matchRange('a','z');
    					break;
    				}
    				case '-':
    				{
    					match('-');
    					break;
    				}
    				default:
    				{
    					if ( _cnt62>=1 ) { break _loop62; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());}
    				}
    				}
    				_cnt62++;
    			} while (true);
    			}
    		}
    		else {
    		}

    		}

    		_ttype = CODE;

    		int newType = testLiteralsTable(CODE);

    		if (newType == CODE)
    		{
    		// we are not really in a statement...
    		env.setInStatement(false);
    		}
    		else if (newType == AELSE || newType == AELSEIF)
    		{
    		// if we are in a global or scoped define, then an &else or &elseif is just text and
    		// we should not change state
    		if (!env.isInDefine())
    		{
    		// we will match an ELSE or ELSEIF - if THEN was emitted (condition was true),
    		// then we need to disable the 'bracesExpand'
    		env.setExpandBraces(!env.isInSkipBlock() && !env.isIfCondValue());
    		}
    		}

    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches a preprocessor input that has no other interpetation.  Combines into one token as many
     * characters as possible without breaking other rules. Breaks at specific characters to avoid
     * lexical non-determinism with other top-level rules.  Comment types {@code STAR_COMMENT} and
     * {@code SLASH_SLASH} are also matched here.  Anything not matched as a comment is returned as
     * a {@code CODE} token.
     */
    	public final void mCODE(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = CODE;
    		int _saveIndex;

    		boolean bogusStmt = false;
    		boolean isComment = false;
    		boolean isMarker  = false;
    		boolean leading   = false;
    		char    marker    = env.getOpt().getMarker();

    		if (LA(1) == '/')
    		{
    		env.setDeferMarkers(true);
    		bogusStmt = true;
    		}

    		// we must safely handle markers in the stream when we see one; the markers are
    		// formatted as follows: <marker_char> <type_char> <data> <marker_char> <space>;
    		// the problem is the trailing space; we normally cannot match a space here but
    		// we must match one (and only one) space just after the 2nd marker char, then
    		// we must turn off isMarker mode; without this change we will exit CODE
    		// unexpectedly which can cause incorrect entering of string mode when we see a
    		// lone embedded " or ' char that is supposed to safely be matched here
    		isMarker = (LA(1) == marker);

    		// a leading marker should be matched AND MUST then exit CODE, because it can be
    		// followed by a string that is not considered CODE; the marker char is configurable
    		// so we can't create a rule to match it separately like COMMENT; a non-leading
    		// marker must not break CODE into chunks because any contained quote chars must
    		// still be eaten
    		leading  = isMarker;

    		{
    		if ((LA(1)=='/') && (LA(2)=='*')) {
    			mSTAR_COMMENT(false);
    			_ttype = STAR_COMMENT; isComment = true;
    		}
    		else if (((LA(1)=='/') && (LA(2)=='/'))&&( !env.isInStatement() )) {
    			mSLASH_SLASH(false);

    			_ttype = SLASH_SLASH;
    			isComment = true;

    		}
    		else if (((LA(1)=='/') && (true))&&( LA(2) != '*' || env.isInStatement() )) {
    			match('/');

    			if (bogusStmt)
    			{
    			env.setDeferMarkers(false);
    			env.clearDeferred();
    			}

    		}
    		else if ((_tokenSet_4.member(LA(1)))) {
    			{
    			match(_tokenSet_4);
    			}
    		}
    		else {
    			throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
    		}

    		}
    		{
    		if (( true )&&( !isComment )) {
    			{
    			_loop69:
    			do {
    				if ((_tokenSet_5.member(LA(1)))) {

    					if (!isMarker)
    					{
    					isMarker = (LA(1) == marker);
    					}

    					{
    					match(_tokenSet_5);
    					}
    				}
    				else if (((LA(1)==' ') && (true))&&( isMarker && !leading )) {
    					match(' ');
    					isMarker = false;
    				}
    				else {
    					break _loop69;
    				}

    			} while (true);
    			}
    		}
    		else {
    		}

    		}
    		{
    		if (((LA(1)==' '))&&( isMarker && leading )) {
    			match(' ');
    		}
    		else {
    		}

    		}
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches Progress language comments, possibly nested. Brackets the comments with a pair of
     * calls to <code>{@link Environment#setInComment(boolean)}</code> so the preprocessor always
     * knows the context.  This supports both the "slash star" and the "slash slash" comment styles.
     * <p>
     * This is a top level lexer rule which means that there is an associated {@code STAR_COMMENT}
     * token.
     */
    	protected final void mSTAR_COMMENT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = STAR_COMMENT;
    		int _saveIndex;

    		commentNesting++;

    		mCOMM_OPEN(false);

    		if (commentNesting == 1)
    		{
    		env.setDeferMarkers(false);
    		env.clearDeferred();
    		env.setInComment(true);
    		}

    		{
    		_loop75:
    		do {
    			// nongreedy exit test
    			if ((LA(1)=='*') && (LA(2)=='/')) break _loop75;
    			if ((LA(1)=='/') && (LA(2)=='*')) {
    				mSTAR_COMMENT(false);
    			}
    			else if ((LA(1)=='\n') && ((LA(2) >= '\u0000' && LA(2) <= '\ufffe'))) {
    				match('\n');
    				newline();
    			}
    			else if (((LA(1) >= '\u0000' && LA(1) <= '\ufffe')) && ((LA(2) >= '\u0000' && LA(2) <= '\ufffe'))) {
    				matchNot(EOF_CHAR);
    			}
    			else {
    				break _loop75;
    			}

    		} while (true);
    		}
    		mCOMM_CLOSE(false);

    		commentNesting--;

    		// reset inComment flag at outermost nesting level only
    		if (commentNesting == 0)
    		{
    		env.setInComment(false);
    		}

    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches the slash slash single-line Progress 4GL comment that will extend to the end
     * of the current line.  No comment nesting occurs in this rule.  If there is a slash star
     * then that is just contents of the slash slash comment.
     */
    	protected final void mSLASH_SLASH(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = SLASH_SLASH;
    		int _saveIndex;

    		env.setDeferMarkers(false);
    		env.clearDeferred();
    		env.setInComment(true);

    		match('/');
    		match('/');
    		{
    		_loop79:
    		do {
    			if ((_tokenSet_6.member(LA(1))) && (true)) {
    				{
    				match(_tokenSet_6);
    				}
    			}
    			else {
    				break _loop79;
    			}

    		} while (true);
    		}

    		env.setInComment(false);

    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches a range of punctuation chars that cause CODE to stop parsing a chunk.  Sets the type
     * to CODE token type.
     */
    	public final void mBREAK_CODE_CHUNKS(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = BREAK_CODE_CHUNKS;
    		int _saveIndex;

    		{
    		switch ( LA(1)) {
    		case '(':
    		{
    			match('(');
    			break;
    		}
    		case ')':
    		{
    			match(')');
    			break;
    		}
    		case '[':
    		{
    			match('[');
    			break;
    		}
    		case ']':
    		{
    			match(']');
    			break;
    		}
    		case ',':
    		{
    			match(',');
    			break;
    		}
    		case '=':
    		{
    			match('=');
    			break;
    		}
    		case '<':
    		{
    			match('<');
    			break;
    		}
    		case '>':
    		{
    			match('>');
    			break;
    		}
    		case '{':
    		{
    			match('{');
    			break;
    		}
    		case '}':
    		{
    			match('}');
    			break;
    		}
    		case '|':
    		{
    			match('|');
    			break;
    		}
    		default:
    		{
    			throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
    		}
    		}
    		}
    		_ttype = CODE;
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches the opening sequence for Progress 4GL comments.
     */
    	protected final void mCOMM_OPEN(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = COMM_OPEN;
    		int _saveIndex;

    		match('/');
    		match('*');
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    /**
     * Matches the closing sequence for Progress 4GL comments.
     */
    	protected final void mCOMM_CLOSE(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
    		int _ttype; Token _token=null; int _begin=text.length();
    		_ttype = COMM_CLOSE;
    		int _saveIndex;

    		match('*');
    		match('/');
    		if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
    			_token = makeToken(_ttype);
    			_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
    		}
    		_returnToken = _token;
    	}

    	private static final long[] mk_tokenSet_0() {
    		long[] data = new long[2048];
    		data[0]=-8070472269077415425L;
    		data[1]=-4035225266795053057L;
    		for (int i = 2; i<=1022; i++) { data[i]=-1L; }
    		data[1023]=9223372036854775807L;
    		return data;
    	}
    	public static final BitSet _tokenSet_0 = new BitSet(mk_tokenSet_0());
    	private static final long[] mk_tokenSet_1() {
    		long[] data = new long[2048];
    		data[0]=-549755814913L;
    		data[1]=-4611686018695823361L;
    		for (int i = 2; i<=1022; i++) { data[i]=-1L; }
    		data[1023]=9223372036854775807L;
    		return data;
    	}
    	public static final BitSet _tokenSet_1 = new BitSet(mk_tokenSet_1());
    	private static final long[] mk_tokenSet_2() {
    		long[] data = new long[2048];
    		data[0]=-17179870209L;
    		data[1]=-4611686018695823361L;
    		for (int i = 2; i<=1022; i++) { data[i]=-1L; }
    		data[1023]=9223372036854775807L;
    		return data;
    	}
    	public static final BitSet _tokenSet_2 = new BitSet(mk_tokenSet_2());
    	private static final long[] mk_tokenSet_3() {
    		long[] data = new long[1025];
    		data[0]=35184372088832L;
    		data[1]=576460743713488896L;
    		return data;
    	}
    	public static final BitSet _tokenSet_3 = new BitSet(mk_tokenSet_3());
    	private static final long[] mk_tokenSet_4() {
    		long[] data = new long[2048];
    		data[0]=-8070613006565770753L;
    		data[1]=-4035225266795053057L;
    		for (int i = 2; i<=1022; i++) { data[i]=-1L; }
    		data[1023]=9223372036854775807L;
    		return data;
    	}
    	public static final BitSet _tokenSet_4 = new BitSet(mk_tokenSet_4());
    	private static final long[] mk_tokenSet_5() {
    		long[] data = new long[2048];
    		data[0]=-8070616837676598785L;
    		data[1]=-4035225266795053057L;
    		for (int i = 2; i<=1022; i++) { data[i]=-1L; }
    		data[1023]=9223372036854775807L;
    		return data;
    	}
    	public static final BitSet _tokenSet_5 = new BitSet(mk_tokenSet_5());
    	private static final long[] mk_tokenSet_6() {
    		long[] data = new long[2048];
    		data[0]=-1025L;
    		for (int i = 1; i<=1022; i++) { data[i]=-1L; }
    		data[1023]=9223372036854775807L;
    		return data;
    	}
    	public static final BitSet _tokenSet_6 = new BitSet(mk_tokenSet_6());

    	}