TextLexer.java
// $ANTLR 2.7.7 (20060906): "text.g" -> "TextLexer.java"$
/*
** Module : TextLexer.java
** TextParser.java
** PreprocTokenTypes.java
** Abstract : This grammar is the top level recognizer of the Progress
** language preprocessor input. It recognizes tabs, nested
** comments, single and double quoted multiline strings and
** the code.
**
** Copyright (c) 2004-2019, Golden Code Development Corporation.
**
** -#- -I- --Date-- -T- --JPRM-- ----------------Description-----------------
** 001 GES 20041116 ADD @18832 WARNING, THIS IS A GENERATED FILE!!!
** DO NOT EDIT THIS FILE. The original source
** file is text.g!
*/
package com.goldencode.p2j.preproc;
import java.io.*;
import java.util.*;
import java.util.logging.*;
import antlr.collections.AST;
import antlr.collections.impl.*;
import antlr.debug.misc.*;
import antlr.*;
import com.goldencode.util.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.util.*;
import java.io.InputStream;
import antlr.TokenStreamException;
import antlr.TokenStreamIOException;
import antlr.TokenStreamRecognitionException;
import antlr.CharStreamException;
import antlr.CharStreamIOException;
import antlr.ANTLRException;
import java.io.Reader;
import java.util.Hashtable;
import antlr.CharScanner;
import antlr.InputBuffer;
import antlr.ByteBuffer;
import antlr.CharBuffer;
import antlr.Token;
import antlr.CommonToken;
import antlr.RecognitionException;
import antlr.NoViableAltForCharException;
import antlr.MismatchedCharException;
import antlr.TokenStream;
import antlr.ANTLRHashString;
import antlr.LexerSharedInputState;
import antlr.collections.impl.BitSet;
import antlr.SemanticException;
/**
* Tokenizes the input stream of characters from the Progress source
* file and returns tokens to the caller according to the needs of
* preprocessor.
* <p>
* The lexer recognizes the current context such as comments, strings
* or code and keeps track of the context switches using
* <code>{@link Environment#setInComment(boolean)}</code> and
* <code>{@link Environment#setInString(boolean)}</code> methods.
*
* @see TextParser
* @see Environment
*/
public class TextLexer extends antlr.CharScanner implements PreprocTokenTypes, TokenStream
{
/** Logger. */
private static final ConversionStatus LOG = ConversionStatus.get(TextLexer.class);
/** keeps the reference to the shared environment */
private Environment env = null;
/** Nesting level for comments. */
private int commentNesting = 0;
/** Control matched quote processing in strings. */
private boolean brokenStringMatching = false;
/**
* Constructor. Creates a lexer attached to the input stream
* taken from the environment. Saves the environment
* for future needs.
*
* @param env
* Shared preprocessor environment.
*/
public TextLexer(Environment env)
{
this(new CharBuffer(env.getIns()));
this.env = env;
this.env.setLsi(this.getInputState());
}
/**
* Expands tabs with spaces right in the ANTLRStringBuilder text.
* This version is used here because according to experiments,
* the tabs expansion for the preprocessor variables is immediate.
*/
public void tab()
{
int c = getColumn();
int nc = (((c-1)/tabsize) + 1) * tabsize + 1; // calculate tab stop
setColumn(nc);
text.setLength(text.length() - 1); // replace '\t' in the buffer
for (; c < nc; c ++) // add more spaces
text.append(' ');
}
/**
* Tests the token text against the literals table and provides
* correct token types for the abbreviated preprocessor statement
* keywords.
*/
public int testLiteralsTable(int ttype)
{
// support abbreviations in some of the possible literals
String[] abbrs =
{
"&global-define",
"&scoped-define",
"&undefine",
"&analyze-suspend",
"&analyze-resume"
};
// minimum lengths of each of the above possible abbreviations
int[] minsz = {5, 5, 6, 10, 10};
String itoken = new String(text.getBuffer(), 0, text.length());
String token = itoken.toLowerCase();
int ltoken = token.length();
for (int i = 0; i < abbrs.length; i ++)
{
if (abbrs[i].startsWith(token) && ltoken >= minsz[i])
{
token = abbrs[i];
}
}
hashString.setString(token);
Integer literalsIndex = (Integer)literals.get(hashString);
if (literalsIndex != null)
{
ttype = literalsIndex.intValue();
}
// disable normal quote matching in STRING until the NL is reached
// (this is necessary since unmatched quotes are allowed in defines
// and all downstream parsing is really broken otherwise)
if (ttype == AGLOBAL || ttype == ASCOPED || ttype == AMESSAGE)
{
brokenStringMatching = true;
}
return ttype;
}
/**
* Display a debug message along with a representation of LA(1).
*
* @param msg
* A debug message.
*/
public void debug(String msg)
throws CharStreamException
{
LOG.log(Level.FINE, String.format("%s '%c' (0x%04X)\n", msg, LA(1), (short) LA(1)));
}
public TextLexer(InputStream in) {
this(new ByteBuffer(in));
}
public TextLexer(Reader in) {
this(new CharBuffer(in));
}
public TextLexer(InputBuffer ib) {
this(new LexerSharedInputState(ib));
}
public TextLexer(LexerSharedInputState state) {
super(state);
caseSensitiveLiterals = false;
setCaseSensitive(false);
literals = new Hashtable();
literals.put(new ANTLRHashString("&endif", this), Integer.valueOf(24));
literals.put(new ANTLRHashString("&analyze-suspend", this), Integer.valueOf(25));
literals.put(new ANTLRHashString("&message", this), Integer.valueOf(18));
literals.put(new ANTLRHashString("&if", this), Integer.valueOf(20));
literals.put(new ANTLRHashString("&scoped-define", this), Integer.valueOf(17));
literals.put(new ANTLRHashString("&global-define", this), Integer.valueOf(16));
literals.put(new ANTLRHashString("&elseif", this), Integer.valueOf(22));
literals.put(new ANTLRHashString("&then", this), Integer.valueOf(21));
literals.put(new ANTLRHashString("&undefine", this), Integer.valueOf(19));
literals.put(new ANTLRHashString("&else", this), Integer.valueOf(23));
literals.put(new ANTLRHashString("&analyze-resume", this), Integer.valueOf(26));
}
public Token nextToken() throws TokenStreamException {
Token theRetToken=null;
tryAgain:
for (;;) {
Token _token = null;
int _ttype = Token.INVALID_TYPE;
resetText();
try { // for char stream error handling
try { // for lexical error handling
switch ( LA(1)) {
case '\t': case ' ':
{
mWS(true);
theRetToken=_returnToken;
break;
}
case '\n':
{
mNL(true);
theRetToken=_returnToken;
break;
}
case '"': case '\'':
{
mSTRING(true);
theRetToken=_returnToken;
break;
}
case '&':
{
mASTMT(true);
theRetToken=_returnToken;
break;
}
case '(': case ')': case ',': case '<':
case '=': case '>': case '[': case ']':
case '{': case '|': case '}':
{
mBREAK_CODE_CHUNKS(true);
theRetToken=_returnToken;
break;
}
default:
if ((_tokenSet_0.member(LA(1)))) {
mCODE(true);
theRetToken=_returnToken;
}
else {
if (LA(1)==EOF_CHAR) {uponEOF(); _returnToken = makeToken(Token.EOF_TYPE);}
else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());}
}
}
if ( _returnToken==null ) continue tryAgain; // found SKIP token
_ttype = _returnToken.getType();
_ttype = testLiteralsTable(_ttype);
_returnToken.setType(_ttype);
return _returnToken;
}
catch (RecognitionException e) {
throw new TokenStreamRecognitionException(e);
}
}
catch (CharStreamException cse) {
if ( cse instanceof CharStreamIOException ) {
throw new TokenStreamIOException(((CharStreamIOException)cse).io);
}
else {
throw new TokenStreamException(cse.getMessage());
}
}
}
}
/**
* Matches any amount of continguous whitespace (spaces and tabs) in a program.
* Newlines are NOT matched.
* <p>
* This is a top level lexer rule which means that there is an associated
* <code>WS</code> token.
*/
public final void mWS(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = WS;
int _saveIndex;
{
int _cnt55=0;
_loop55:
do {
switch ( LA(1)) {
case ' ':
{
match(' ');
break;
}
case '\t':
{
match('\t');
break;
}
default:
{
if ( _cnt55>=1 ) { break _loop55; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());}
}
}
_cnt55++;
} while (true);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches a single newline character.
* <p>
* This is a top level lexer rule which means that there is an associated
* <code>NL</code> token.
*/
public final void mNL(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = NL;
int _saveIndex;
match('\n');
newline();
// if a scoped or global define has ended, re-enable STRING processing
if (brokenStringMatching)
{
brokenStringMatching = false;
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches Progress language strings. Brackets the string with a pair
* of calls to <code>{@link Environment#setInString(boolean)}</code>
* so the preprocessor always knows the context.
* <p>
* Depending on -keeptildes preprocessor options, newlines may be required
* to be deleted from the strings. This requirement does not extend to the
* characters which are coded as escape sequences, though. The final action in
* this rule postprocesses the string contents so they comply with the
* requirement.
* <p>
* To tell the original newlines from the escaped ones, ClearStream uses the
* following technique. For every converted character, that becomes newline,
* an escape character is inserted in front of it into the stream. The
* postprocessing action is supposed to detect this escape character, discard
* it but leave the newline that follows untouched. The original newlines come
* without escape characters and are deleted.
* <p>
* For this technique to work, an escape character has to be assigned which
* normally can't be seen in the stream. Such a character is CR, due to the
* fact that ClearStream strips off all original CRs and replaces them with
* NLs. But the same problem of converted CRs still exists. Fortunately, it is
* solved in a similar way. As the result, the escape character, CR, is
* inserted in front of every converted CR or NL. The valid escape sequences
* that ClearStream produces are CR CR and CR NL. In both cases, the first CR
* has to be removed from the stream and the second character left untouched.
* <p>
* One more complication is due to the fact that there is no on time state
* change signalling between the lexer and the ClearStream code. Although this
* rule signals the end of string by calling <code>inString(false)</code>, the
* call is one character late because of the k=2 lookahead in the lexer. Late
* signalling leaves a hole in the algorithm for a wrong escape character
* insertion just outside the string, which should not happen. Consider the
* following stream:
* <pre>
* "..."~n~n
* </pre>
* Due to the late signalling, ClearStream reads and converts the ~n Progress
* escape sequence before this rule signals the end of string. As the result,
* ClearStream produces the following stream:
* <pre>
* "..." CR NL NL
* </pre>
* Notice that only the first NL gets escaped with CR, the second NL comes
* when the end of string signal has been received. This issue is fixed by
* checking the next character in the lookahead buffer LA(1) in the
* postprocessing rule. If it is CR, it has to be discarded, as it only can be
* there as the result of this late signalling.
* <p>
* This is a top level lexer rule which means that there is an associated
* <code>STRING</code> token.
* <p>
* If there is an escaped <code>null</code> character in the string literal,
* it must be "space-ified" by converting it and all following characters
* into spaces (including the proper reduction of other following escape
* sequences into a single space character). One can encode a
* <code>null</code> escape sequence in a string literal in Progress BUT a
* <code>null</code> character will NEVER end up in the result! This method
* duplicates that processing.
*
* @throws antlr.RecognitionException
* @throws antlr.CharStreamException
* @throws antlr.TokenStreamException
*/
public final void mSTRING(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = STRING;
int _saveIndex;
if (!brokenStringMatching)
{
env.setInString(true);
}
{
switch ( LA(1)) {
case '\'':
{
mASTRING(false);
break;
}
case '"':
{
mQSTRING(false);
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
}
}
}
if (!brokenStringMatching)
{
env.setInString(false);
text.setLength(Preprocessor.stripNL(text.getBuffer(),
text.length(),
env.getOpt().isKeepTildes()));
if (LA(1) == 0x0D)
{
consume();
text.setLength(text.length() - 1);
}
boolean unixEsc = env.getOpt().isUnixEscapes();
// implement spacifying transformation here
setText(character.progressSpacifyNull(getText(), unixEsc));
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches an opening single quote, arbitrary contents and an ending single
* quote. Two continguous single quote characters and an escape prefixed
* single quote character are accepted as contents (they do not terminate
* the string).
* <p>
* Any newlines inside the string are identified and the lexer's internal
* newline counter is properly maintained. Note that all such newlines
* are maintained in the output string because these are the escaped chars
* that have been left behind by the preprocessor. If such characters are not
* escaped, then the preprocessor removes such chars (carriage returns and
* line feeds). This is how Progress 4GL handles these characters. This
* means that in a raw source file that has not been preprocessed, a string
* literal can be split across any number of lines and the Progress
* preprocessor will put the string back together, ignoring the carriage
* returns and newlines.
* <p>
* Tilde and the backslash can BOTH be Progress escape characters and as such,
* they need special attention. Backslash is only honored if UNIX escapes
* mode is set on. In particular, this rule must separately match 3 or 6
* constructs as part of a string depending on which escape sequences are
* honored.
* <p>
* <pre>
* ~'
* \'
* ~~
* ~\
* \\
* \~
* ~
* \
* </pre>
* <p>
* The first 2 are a way of embedding a quote in a string. The next 4 are
* important because if this rule were to encounter '~~', '\\', '~\' or '\~'
* (which are valid strings) the rule would consume the first escape and then
* encounter an escaped quote which would not terminate the string properly,
* leading to a non-ending string. So this rule matches on any escaped escape
* char to eliminate this situation. Finally, a single ~ or \ that is not
* followed by a " or a duplicate escape char is matched as a single
* character. This is required (one may think that the closure rule
* should handle this case) because when ANTLR sees a ~ or \ in the leftmost
* position of the alternatives, it DOES NOT include it in the list of
* 'everything that is not a closing quote'. Likewise \n is not included.
* Of course, these tilde constructions must be placed in a specific order
* and if so, the ambiguity warnings that ANTLR reports can be disabled.
* <p>
* The greedy option does not need to be disabled here as the closure rule
* termination is built into the subrule itself: it accepts anything that
* isn't an unescaped single quote character (see above).
* <p>
* Tabs and spaces are maintained inside strings.
*/
protected final void mASTRING(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = ASTRING;
int _saveIndex;
boolean honor = env.getOpt().isUnixEscapes();
{
match('\'');
{
_loop85:
do {
if ((LA(1)=='~') && (LA(2)=='\'')) {
match("~\'");
}
else if ((LA(1)=='~') && (LA(2)=='~')) {
match("~~");
}
else if ((LA(1)=='~') && (LA(2)=='\\')) {
match("~\\");
}
else if (((LA(1)=='\\') && (LA(2)=='\''))&&( honor )) {
match("\\\'");
}
else if (((LA(1)=='\\') && (LA(2)=='\\'))&&( honor )) {
match("\\\\");
}
else if (((LA(1)=='\\') && (LA(2)=='~'))&&( honor )) {
match("\\~");
}
else if ((LA(1)=='\'') && (LA(2)=='\'')) {
match("\'\'");
}
else if ((LA(1)=='~') && (true)) {
match('~');
}
else if ((LA(1)=='\\') && (true)) {
match('\\');
}
else if (((LA(1)=='\n'))&&( !brokenStringMatching )) {
match('\n');
newline();
}
else if ((_tokenSet_1.member(LA(1)))) {
matchNot('\'');
}
else {
break _loop85;
}
} while (true);
}
{
if (((LA(1)=='\'') && (true))&&( !brokenStringMatching )) {
match('\'');
}
else if (( true )&&( brokenStringMatching )) {
{
if ((LA(1)=='\'')) {
match('\'');
}
else {
}
}
}
else {
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
}
}
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches an opening double quote, arbitrary contents and an ending double
* quote. Two continguous double quote characters and an escape prefixed
* double quote character are accepted as contents (they do not terminate
* the string).
* <p>
* Any newlines inside the string are identified and the lexer's internal
* newline counter is properly maintained. Note that all such newlines
* are maintained in the output string because these are the escaped chars
* that have been left behind by the preprocessor. If such characters are not
* escaped, then the preprocessor removes such chars (carriage returns and
* line feeds). This is how Progress 4GL handles these characters. This
* means that in a raw source file that has not been preprocessed, a string
* literal can be split across any number of lines and the Progress
* preprocessor will put the string back together, ignoring the carriage
* returns and newlines.
* <p>
* Tilde and the backslash can BOTH be Progress escape characters and as such,
* they need special attention. Backslash is only honored if UNIX escapes
* mode is set on. In particular, this rule must separately match 3 or 6
* constructs as part of a string depending on which escape sequences are
* honored.
* <p>
* <pre>
* ~"
* \"
* ~~
* ~\
* \\
* \~
* ~
* \
* </pre>
* <p>
* The first 2 are a way of embedding a quote in a string. The next 4 are
* important because if this rule were to encounter '~~', '\\', '~\' or '\~'
* (which are valid strings) the rule would consume the first escape and then
* encounter an escaped quote which would not terminate the string properly,
* leading to a non-ending string. So this rule matches on any escaped escape
* char to eliminate this situation. Finally, a double ~ or \ that is not
* followed by a " or a duplicate escape char is matched as a double
* character. This is required (one may think that the closure rule
* should handle this case) because when ANTLR sees a ~ or \ in the leftmost
* position of the alternatives, it DOES NOT include it in the list of
* 'everything that is not a closing quote'. Likewise \n is not included.
* Of course, these tilde constructions must be placed in a specific order
* and if so, the ambiguity warnings that ANTLR reports can be disabled.
* <p>
* The greedy option does not need to be disabled here as the closure rule
* termination is built into the subrule itself: it accepts anything that
* isn't an unescaped double quote character (see above).
* <p>
* Tabs and spaces are maintained inside strings.
*/
protected final void mQSTRING(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = QSTRING;
int _saveIndex;
boolean honor = env.getOpt().isUnixEscapes();
{
match('\"');
{
_loop91:
do {
if ((LA(1)=='~') && (LA(2)=='"')) {
match("~\"");
}
else if ((LA(1)=='~') && (LA(2)=='~')) {
match("~~");
}
else if ((LA(1)=='~') && (LA(2)=='\\')) {
match("~\\");
}
else if (((LA(1)=='\\') && (LA(2)=='"'))&&( honor )) {
match("\\\"");
}
else if (((LA(1)=='\\') && (LA(2)=='\\'))&&( honor )) {
match("\\\\");
}
else if (((LA(1)=='\\') && (LA(2)=='~'))&&( honor )) {
match("\\~");
}
else if ((LA(1)=='"') && (LA(2)=='"')) {
match("\"\"");
}
else if ((LA(1)=='~') && (true)) {
match('~');
}
else if ((LA(1)=='\\') && (true)) {
match('\\');
}
else if (((LA(1)=='\n'))&&( !brokenStringMatching )) {
match('\n');
newline();
}
else if ((_tokenSet_2.member(LA(1)))) {
matchNot('\"');
}
else {
break _loop91;
}
} while (true);
}
{
if (((LA(1)=='"') && (true))&&( !brokenStringMatching )) {
match('\"');
}
else if (( true )&&( brokenStringMatching )) {
{
if ((LA(1)=='"')) {
match('\"');
}
else {
}
}
}
else {
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
}
}
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches any ampersand prefaced symbolic name including all recognized or
* unrecognized preprocessor directives. Such a statement starts with an
* <code>'&'</code> followed by letters and possibly the hyphen character.
* <p>
* The resulting token's text will be compared with the literals table by
* the <code>testLiteralsTable</code> method of this class. The matching is
* done case-insensitively and a subset of the symbols can be matched with
* an abbreviated form. The token type replacement occurs as follows:
* <p>
* <pre>
* Token Type Matched Text Minimum Abbreviation Chars
* ----------- ------------------ --------------------------
* AGLOBAL &global-define 5
* ASCOPED &scoped-define 5
* AMESSAGE &message n/a
* AUNDEFINE &undefine 6
* AIF &if n/a
* ATHEN &then n/a
* AELSEIF &elseif n/a
* AELSE &else n/a
* AENDIF &endif n/a
* ASUSPEND &analyze-suspend n/a
* ARESUME &analyze-resume n/a
* </pre>
* <p>
* This is a top level lexer rule which means that in the case where the
* statement is unrecognized, there will be an associated <code>ASTMT</code>
* token.
* <p>
* If the leading ampersand is not followed by a recognized statement, then
* the ampersand and any following alphabetic characters will be returned as a
* <code>CODE</code> token. It is also possible to return the ampersand by
* itself as a <code>CODE</code> token, if there are no following alphabetic
* characters. Both of these combinations are possible in certain 4GL parsing
* cases like shell commands the ampersand character is perfectly valid. Another
* example is that ampersands can appear inside a variable name like my-p&l-var.
* The 4GL is just insane.
*
* @throws antlr.RecognitionException
* @throws antlr.CharStreamException
* @throws antlr.TokenStreamException
*/
public final void mASTMT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = ASTMT;
int _saveIndex;
// we used to set this after we matched the ampersand, but it turns out that there can be
// constructs like &{&METADEFINE}-define where {&METADEFINE} expands to global or scoped
// for which the 4GL happily treats the cumulative result as &global-define or
// &scoped-define, this was broken when in-stmt was set too late because the next
// characters read were marker chars not the expanded text; by setting this earlier
// we eliminate the marker chars and everything works as in the 4GL
env.setInStatement(true);
match('&');
{
if ((_tokenSet_3.member(LA(1)))) {
{
int _cnt62=0;
_loop62:
do {
switch ( LA(1)) {
case 'a': case 'b': case 'c': case 'd':
case 'e': case 'f': case 'g': case 'h':
case 'i': case 'j': case 'k': case 'l':
case 'm': case 'n': case 'o': case 'p':
case 'q': case 'r': case 's': case 't':
case 'u': case 'v': case 'w': case 'x':
case 'y': case 'z':
{
matchRange('a','z');
break;
}
case '-':
{
match('-');
break;
}
default:
{
if ( _cnt62>=1 ) { break _loop62; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());}
}
}
_cnt62++;
} while (true);
}
}
else {
}
}
_ttype = CODE;
int newType = testLiteralsTable(CODE);
if (newType == CODE)
{
// we are not really in a statement...
env.setInStatement(false);
}
else if (newType == AELSE || newType == AELSEIF)
{
// if we are in a global or scoped define, then an &else or &elseif is just text and
// we should not change state
if (!env.isInDefine())
{
// we will match an ELSE or ELSEIF - if THEN was emitted (condition was true),
// then we need to disable the 'bracesExpand'
env.setExpandBraces(!env.isInSkipBlock() && !env.isIfCondValue());
}
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches a preprocessor input that has no other interpetation. Combines into one token as many
* characters as possible without breaking other rules. Breaks at specific characters to avoid
* lexical non-determinism with other top-level rules. Comment types {@code STAR_COMMENT} and
* {@code SLASH_SLASH} are also matched here. Anything not matched as a comment is returned as
* a {@code CODE} token.
*/
public final void mCODE(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = CODE;
int _saveIndex;
boolean bogusStmt = false;
boolean isComment = false;
boolean isMarker = false;
boolean leading = false;
char marker = env.getOpt().getMarker();
if (LA(1) == '/')
{
env.setDeferMarkers(true);
bogusStmt = true;
}
// we must safely handle markers in the stream when we see one; the markers are
// formatted as follows: <marker_char> <type_char> <data> <marker_char> <space>;
// the problem is the trailing space; we normally cannot match a space here but
// we must match one (and only one) space just after the 2nd marker char, then
// we must turn off isMarker mode; without this change we will exit CODE
// unexpectedly which can cause incorrect entering of string mode when we see a
// lone embedded " or ' char that is supposed to safely be matched here
isMarker = (LA(1) == marker);
// a leading marker should be matched AND MUST then exit CODE, because it can be
// followed by a string that is not considered CODE; the marker char is configurable
// so we can't create a rule to match it separately like COMMENT; a non-leading
// marker must not break CODE into chunks because any contained quote chars must
// still be eaten
leading = isMarker;
{
if ((LA(1)=='/') && (LA(2)=='*')) {
mSTAR_COMMENT(false);
_ttype = STAR_COMMENT; isComment = true;
}
else if (((LA(1)=='/') && (LA(2)=='/'))&&( !env.isInStatement() )) {
mSLASH_SLASH(false);
_ttype = SLASH_SLASH;
isComment = true;
}
else if (((LA(1)=='/') && (true))&&( LA(2) != '*' || env.isInStatement() )) {
match('/');
if (bogusStmt)
{
env.setDeferMarkers(false);
env.clearDeferred();
}
}
else if ((_tokenSet_4.member(LA(1)))) {
{
match(_tokenSet_4);
}
}
else {
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
}
}
{
if (( true )&&( !isComment )) {
{
_loop69:
do {
if ((_tokenSet_5.member(LA(1)))) {
if (!isMarker)
{
isMarker = (LA(1) == marker);
}
{
match(_tokenSet_5);
}
}
else if (((LA(1)==' ') && (true))&&( isMarker && !leading )) {
match(' ');
isMarker = false;
}
else {
break _loop69;
}
} while (true);
}
}
else {
}
}
{
if (((LA(1)==' '))&&( isMarker && leading )) {
match(' ');
}
else {
}
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches Progress language comments, possibly nested. Brackets the comments with a pair of
* calls to <code>{@link Environment#setInComment(boolean)}</code> so the preprocessor always
* knows the context. This supports both the "slash star" and the "slash slash" comment styles.
* <p>
* This is a top level lexer rule which means that there is an associated {@code STAR_COMMENT}
* token.
*/
protected final void mSTAR_COMMENT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = STAR_COMMENT;
int _saveIndex;
commentNesting++;
mCOMM_OPEN(false);
if (commentNesting == 1)
{
env.setDeferMarkers(false);
env.clearDeferred();
env.setInComment(true);
}
{
_loop75:
do {
// nongreedy exit test
if ((LA(1)=='*') && (LA(2)=='/')) break _loop75;
if ((LA(1)=='/') && (LA(2)=='*')) {
mSTAR_COMMENT(false);
}
else if ((LA(1)=='\n') && ((LA(2) >= '\u0000' && LA(2) <= '\ufffe'))) {
match('\n');
newline();
}
else if (((LA(1) >= '\u0000' && LA(1) <= '\ufffe')) && ((LA(2) >= '\u0000' && LA(2) <= '\ufffe'))) {
matchNot(EOF_CHAR);
}
else {
break _loop75;
}
} while (true);
}
mCOMM_CLOSE(false);
commentNesting--;
// reset inComment flag at outermost nesting level only
if (commentNesting == 0)
{
env.setInComment(false);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches the slash slash single-line Progress 4GL comment that will extend to the end
* of the current line. No comment nesting occurs in this rule. If there is a slash star
* then that is just contents of the slash slash comment.
*/
protected final void mSLASH_SLASH(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = SLASH_SLASH;
int _saveIndex;
env.setDeferMarkers(false);
env.clearDeferred();
env.setInComment(true);
match('/');
match('/');
{
_loop79:
do {
if ((_tokenSet_6.member(LA(1))) && (true)) {
{
match(_tokenSet_6);
}
}
else {
break _loop79;
}
} while (true);
}
env.setInComment(false);
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches a range of punctuation chars that cause CODE to stop parsing a chunk. Sets the type
* to CODE token type.
*/
public final void mBREAK_CODE_CHUNKS(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = BREAK_CODE_CHUNKS;
int _saveIndex;
{
switch ( LA(1)) {
case '(':
{
match('(');
break;
}
case ')':
{
match(')');
break;
}
case '[':
{
match('[');
break;
}
case ']':
{
match(']');
break;
}
case ',':
{
match(',');
break;
}
case '=':
{
match('=');
break;
}
case '<':
{
match('<');
break;
}
case '>':
{
match('>');
break;
}
case '{':
{
match('{');
break;
}
case '}':
{
match('}');
break;
}
case '|':
{
match('|');
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());
}
}
}
_ttype = CODE;
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches the opening sequence for Progress 4GL comments.
*/
protected final void mCOMM_OPEN(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = COMM_OPEN;
int _saveIndex;
match('/');
match('*');
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
/**
* Matches the closing sequence for Progress 4GL comments.
*/
protected final void mCOMM_CLOSE(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = COMM_CLOSE;
int _saveIndex;
match('*');
match('/');
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
private static final long[] mk_tokenSet_0() {
long[] data = new long[2048];
data[0]=-8070472269077415425L;
data[1]=-4035225266795053057L;
for (int i = 2; i<=1022; i++) { data[i]=-1L; }
data[1023]=9223372036854775807L;
return data;
}
public static final BitSet _tokenSet_0 = new BitSet(mk_tokenSet_0());
private static final long[] mk_tokenSet_1() {
long[] data = new long[2048];
data[0]=-549755814913L;
data[1]=-4611686018695823361L;
for (int i = 2; i<=1022; i++) { data[i]=-1L; }
data[1023]=9223372036854775807L;
return data;
}
public static final BitSet _tokenSet_1 = new BitSet(mk_tokenSet_1());
private static final long[] mk_tokenSet_2() {
long[] data = new long[2048];
data[0]=-17179870209L;
data[1]=-4611686018695823361L;
for (int i = 2; i<=1022; i++) { data[i]=-1L; }
data[1023]=9223372036854775807L;
return data;
}
public static final BitSet _tokenSet_2 = new BitSet(mk_tokenSet_2());
private static final long[] mk_tokenSet_3() {
long[] data = new long[1025];
data[0]=35184372088832L;
data[1]=576460743713488896L;
return data;
}
public static final BitSet _tokenSet_3 = new BitSet(mk_tokenSet_3());
private static final long[] mk_tokenSet_4() {
long[] data = new long[2048];
data[0]=-8070613006565770753L;
data[1]=-4035225266795053057L;
for (int i = 2; i<=1022; i++) { data[i]=-1L; }
data[1023]=9223372036854775807L;
return data;
}
public static final BitSet _tokenSet_4 = new BitSet(mk_tokenSet_4());
private static final long[] mk_tokenSet_5() {
long[] data = new long[2048];
data[0]=-8070616837676598785L;
data[1]=-4035225266795053057L;
for (int i = 2; i<=1022; i++) { data[i]=-1L; }
data[1023]=9223372036854775807L;
return data;
}
public static final BitSet _tokenSet_5 = new BitSet(mk_tokenSet_5());
private static final long[] mk_tokenSet_6() {
long[] data = new long[2048];
data[0]=-1025L;
for (int i = 1; i<=1022; i++) { data[i]=-1L; }
data[1023]=9223372036854775807L;
return data;
}
public static final BitSet _tokenSet_6 = new BitSet(mk_tokenSet_6());
}