# Environment.java

    /*
    ** Module   : Environment.java
    ** Abstract : Encapsulates the shared environment for parsers, lexers and drivers.
    **
    ** Copyright (c) 2004-2020, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ---------------------------------Description-----------------------------------
    ** 001 NVS 20041109   @18765 Created. Environment class is a collection
    **                           of references and methods giving access to
    **                           all objects shared between the ANTLR
    **                           generated code and the manually written code.
    ** 002 NVS 20050207   @19629 Added support for dynamic arguments parsing.
    **                           Environment class now provides a private
    **                           member that may point to a Map with parsed
    **                           arguments, and setMap and getMap methods.
    ** 003 NVS 20050413   @20689 Inserted a TO DO marker around System.exit().
    ** 004 NVS 20050510   @21147 System.exit() replaced with the IOException
    **                           in case there is no room in the pushback
    **                           buffer.
    **                           Added line and column information tracking
    **                           for the preprocessor output stream. Added
    **                           methods to set and get line and column
    **                           numbers. Added markers support code.
    ** 005 NVS 20050517   @21214 handleMarker throws an exception when used as
    **                           -lexonly. Made that code conditional.
    **                           Fixed marker handling bug. Markers now
    **                           include a trailing space. See ##21210-21212
    **                           for details.
    ** 006 NVS 20050518   @21237 Simplified the marker handling.
    ** 007 NVS 20050526   @21311 javadoc fixes
    ** 008 GES 20071204   @36193 Added a flag to disable the expansions done
    **                           by the braces parser. Minor refactoring for
    **                           code reduction. Added defer marker processing
    **                           flag to help control braces expansion. Added
    **                           a helper for obtaining lookahead chars.
    ** 009 GES 20080308   @37647 Switched to using character readers and
    **                           writers instead of byte streams. This handles
    **                           I18N needs.
    ** 010 GES 20081010   @40334 Removed unnecessary throws clauses.
    ** 011 GES 20090424   @41949 Import change.
    ** 012 GES 20110628          Lexer multiplexing/switching is no longer needed.
    **                           Fixed bug in lookahead() where wrong text is seen
    **                           because the syncConsume() was not called in some
    **                           cases.
    ** 013 GES 20110901          Added warning message support.
    ** 014 EVL 20160223          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
    ** 015 CA  20170825          Collect all preprocessor symbols via addConstantDefinitions.
    ** 016 CA  20170831          In ASTMT, the lexer must not set 'inStatement' set unless what we
    **                           are matching really is a statement.
    **                           If a AELSE or AELIF is matched with IF condition true (i.e.THEN
    **                           branch is executed), disable the brace expansio, as the body will
    **                           not execute.
    ** 017 GES 20180926          Added inDefine flag.
    ** 018 GES 20200306          Minor code cleanup and dead code removal.
    ** 019 GES 20200625          Created common helper to calculate push vs queue for markers. Added code to
    **                           detect unbalanced end of include file markers.
    **     GES 20200629          Made the unbalanced eoi into a warning (an error is too much since this
    **                           condition can be ignored).
    ** 020 CA  20201015          Replaced java.util.Stack with ArrayDeque (as synchronization is not required).
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.preproc;

    import com.goldencode.util.*;
    import com.goldencode.p2j.util.*;
    import java.io.*;
    import java.util.*;
    import javax.xml.parsers.ParserConfigurationException;

    import antlr.collections.AST;
    import antlr.collections.impl.*;
    import antlr.debug.misc.*;
    import antlr.*;

    /**
     * Encapsulates the shared environment for parsers, lexers and drivers.
     * There must be an instance of the class shared between parsers and lexers
     * if they are to work on the same input and output. However, multiple
     * environments can be instantiated to conduct multiple unrelated "threads"
     * of work with their independent parsers and lexers, if necessary.
     * <p>
     * An instance of the Environment is given as a parameter to constructors
     * of parsers and lexers. This is how they can access this information
     * later in grammar actions.
     * <p>
     * This class also provides a number of useful shortcuts for method calls.
     * These shortcuts reduce the verbosity of the grammar actions and make
     * the grammar files more readable.
     * <p>
     * The latest revision of this class maintains the line and column information
     * for the output stream. These can be read and set. This functionality is
     * required for tracking the portions of the output generated by a specific
     * include file.
     */
    public class Environment
    {
       /** Parent node or <code>null</code>. */
       private Environment parent = null;

       /** Top level (parent of all parents) or <code>null</code>. */
       private Environment rootEnv = null;

       /** Type of the environment - file (true) or reference (false). */
       private boolean file = true;

       /** ANTLR lexer shared input state object */
       private LexerSharedInputState lsi = null;

       /** ANTLR parser shared input state object */
       private ParserSharedInputState psi = null;

       /** reference to a lexer for calling <code>newline()</code> etc */
       private CharScanner lexer = null;

       /** reference to a parser for calling <code>setFileName()</code> */
       private Parser parser = null;

       /** input stream with a large pushback buffer */
       private ClearStream ins = null;

       /** Stream for preprocessor output. */
       private Writer ous = null;

       /** current line number for the preprocessor output */
       private int lineNo = 1;

       /** current column number for the preprocessor output */
       private int colNo = 1;

       /** Stream for error messages. */
       private PrintWriter err = null;

       /** Stream for warning messages. */
       private PrintWriter warn = null;

       /** Scoped symbol dictionary. */
       private ScopedSymbolDictionary sym = null;

       /** preprocessor options */
       private Options opt = null;

       /** counter for the preprocessor errors */
       private int errors = 0;

       /** Warnings counter. */
       private int warnings = 0;

       /** reference to a Map used to parse arguments only */
       private Map args = null;

       /** Configurable marker character.
        *  Markers are inserted into the input stream to indicate boundaries of
        *  included files, references etc.
        */
       private char marker = '\001';

       /** reference to Hints */
       private Hints hints = null;

       /** buffered reference hint markers */
       private StringBuffer refMarks = null;

       /** <code>true</code> if in a preprocessor statement. */
       private boolean inStat = false;

       /** <code>true</code> if space removal is pending. */
       private boolean pendingSpaceRemoval = false;

       /** Disable/enable braces parser expansions. */
       private boolean expandBraces = true;

       /** Flag to identify when we are processing a global or scoped definition. */
       private boolean inDefine = false;

       /** Queue markers instead of pushing them back on the input stream. */
       private boolean deferMarkers = false;

       /** The value of the last executed IF or ELSEIF statement. */
       private boolean ifCondValue = false;

       /** Flag indicating the parser is operating a skipBlock. */
       private boolean inSkipBlock = false;

       /**
        * Construct an instance.
        *
        * @param    parent
        *           Parent Environment object or <code>null</code>.
        * @param    ins
        *           Input stream, an instance of ClearStream providing
        *           input for all lexers using this environment.
        * @param    ous
        *           Stream for preprocessor output.
        * @param    err
        *           Stream for preprocessor error messages.
        * @param    warn
        *           Stream for preprocessor warning messages.
        * @param    sym
        *           Scoped symbol dictionary.
        * @param    opt
        *           Configuration options.
        *
        * @throws   ParserConfigurationException
        *           Forwarded from XmlHelper.
        * @throws   IOException
        *           Forwarded from XmlHelper.
        */
       public Environment(Environment            parent,
                          ClearStream            ins,
                          Writer                 ous,
                          Writer                 err,
                          Writer                 warn,
                          ScopedSymbolDictionary sym,
                          Options                opt)
       throws ParserConfigurationException,
              IOException
       {
          this.parent = parent;
          this.ins    = ins;
          this.ous    = ous;
          this.sym    = sym;
          this.opt    = opt;

          if (err != null)
          {
             this.err = (err instanceof PrintWriter) ? (PrintWriter) err
                                                     : new PrintWriter(err);
             this.warn = this.err;
          }

          if (warn != null)
          {
             this.warn = (warn instanceof PrintWriter) ? (PrintWriter) warn
                                                       : new PrintWriter(warn);
          }

          if (parent != null)
          {
             // propagate the parents braces expansion setting if there is a
             // parent
             expandBraces = parent.expandBraces;

             Environment next = parent;

             while (next != null)
             {
                rootEnv = next;
                next = next.parent;
             }
          }

          marker = opt.getMarker();

          if (opt.getHintsFile() != null)
          {
             hints = new Hints(opt.getHintsFile());
             refMarks = new StringBuffer();
          }
       }

       /**
        * Add all preprocessor level symbol definitions into the hints, for the current scope.
        */
       public void addConstantDefinitions()
       {
          if (hints == null)
          {
             return;
          }

          Map<?, ?> dict = sym.getDictionaryAtScope(0, false);

          for (Object key : dict.keySet())
          {
             if (key instanceof String)
             {
                Object val = dict.get(key);

                if (val instanceof Symbol)
                {
                   Symbol b = (Symbol) val;

                   hints.createConstantSymbol((String) key, b);
                }
             }
          }
       }

       /**
        * Sets the environment type.
        *
        * @param    file
        *           <code>true</code> if this environment is for a file.
        *           Otherwise is a reference.
        */
       public void setFile(boolean file)
       {
          this.file = file;
       }

       /**
        * Gets the environment type.
        *
        * @return   <code>true</code> if this environment is for a file.
        *           Otherwise is a reference.
        */
       public boolean isFile()
       {
          return file;
       }

       /**
        * Gets the parent environment.
        *
        * @return parent Environment object or <code>null</code>.
        */
       public Environment getParent()
       {
          return parent;
       }

       /**
        * Report if markers should be pushed or queued.  Pushing writes the markers into the stream immediately
        * where queuing defers the processing of markers until a safe time.
        *
        * @return   {@code true} if markers should be pushed.
        */
       public boolean isPush()
       {
          return isFile() && !isInStatement() && !isDeferMarkers();
       }

       /**
        * Returns the LexerSharedInputState object.
        *
        * @return ANTLR LexerSharedInputState associated with
        *         this Environment
        */
       public LexerSharedInputState getLsi()
       {
          return lsi;
       }

       /**
        * Sets the LexerSharedInputState object.
        *
        * @param  lsi
        *         ANTLR LexerSharedInputState to be associated with
        *         this Environment
        */
       public void setLsi(LexerSharedInputState lsi)
       {
          this.lsi = lsi;
       }

       /**
        * Returns the ParserSharedInputState object.
        *
        * @return ANTLR ParserSharedInputState associated with
        *         this Environment
        */
       public ParserSharedInputState getPsi()
       {
          return psi;
       }

       /**
        * Sets the ParserSharedInputState object.
        *
        * @param  psi
        *         ANTLR ParserSharedInputState to be associated with
        *         this Environment
        */
       public void setPsi(ParserSharedInputState psi)
       {
          this.psi = psi;
       }

       /**
        * Obtain the lookahead of the given or current lexer as a string. If the
        * result is the single character <code>EOF</code> or if there is no
        * lookahead, the empty string is returned.
        *
        * @param    lexer
        *           The lexer from which to obtain the lookahead or
        *           <code>null</code> to use the currently defined lexer within
        *           this instance.
        *
        * @return   Buffered characters already read from the stream.
        */
       public String lookahead(CharScanner lexer)
       {
          if (lexer == null)
          {
             lexer = this.lexer;
          }

          InputBuffer inBuf = lexer.getInputBuffer();

          // we don't really need to "mark", but it is harmless and calls
          // syncConsume() which is VERY important because otherwise the LA
          // chars buffer will still have unconsumed content
          inBuf.mark();

          // it is now safe to get the lookahead
          String buffered = inBuf.getLAChars();

          // clear the "mark"
          inBuf.commit();

          if (buffered == null)
          {
             buffered = "";
          }
          else if (buffered.length() == 1)
          {
             // check for EOF
             if (buffered.charAt(0) == 0xFFFF)
             {
                // don't push EOF back
                buffered = "";
             }
          }

          return buffered;
       }

       /**
        * Attaches the specified lexer to this environment.
        *
        * @param  lexer
        *         ANTLR CharScanner lexer to be attached to
        *         this Environment
        */
       public void setLex(CharScanner lexer)
       {
          this.lexer = lexer;
       }

       /**
        * Returns the lexer attached to this environment.
        *
        * @return
        *         ANTLR CharScanner lexer attached to this Environment
        */
       public CharScanner getLex()
       {
          return lexer;
       }

       /**
        * Attaches the specified parser to this environment.
        *
        * @param  parser
        *         ANTLR Parser to be attached to this Environment
        */
       public void setPar(Parser parser)
       {
          this.parser = parser;
       }

       /**
        * Returns the parser attached to this environment.
        *
        * @return
        *         ANTLR Parser attached to this Environment
        */
       public Parser getPar()
       {
          return parser;
       }

       /**
        * Returns the input source.
        *
        * @return   The input source for this environment.
        */
       public Reader getIns()
       {
          return ins;
       }

       /**
        * Sets the inComment state.
        *
        * @param  inComment
        *         new state of the flag
        */
       public void setInComment(boolean inComment)
       {
          ins.setInComment(inComment);
       }

       /**
        * Sets the inString state.
        *
        * @param  inString
        *         new state of the flag
        */
       public void setInString(boolean inString)
       {
          ins.setInString(inString);
       }

       /**
        * Detects if the processing is inside a string literal.
        *
        * @return   <code>true</code> if the current processing is inside a
        *           string literal.
        */
       public boolean isInString()
       {
          return ins.isInString();
       }

       /**
        * Sets the inStatement state.
        *
        * @param  inStat
        *         new state of the flag
        */
       public void setInStatement(boolean inStat)
       {
          this.inStat = inStat;
       }

       /**
        * Gets the inStatement state.
        *
        * @return <code>true</code> if in statement flag is set
        */
       public boolean isInStatement()
       {
          return (rootEnv != null) ? rootEnv.inStat : inStat;
       }

       /**
        * Sets the flag to enable or disable the braces parser's expansion of
        * braces.
        *
        * @param    expandBraces
        *           <code>true</code> to enable.
        */
       public void setExpandBraces(boolean expandBraces)
       {
          this.expandBraces = expandBraces;
       }

       /**
        * Gets the flag that enables or disables the braces parser's expansion of
        * braces.
        *
        * @return   <code>true</code> if braces expansion is enabled.
        */
       public boolean isExpandBraces()
       {
          return expandBraces;
       }

       /**
        * Sets the flag that identifies if we are currently processing a global or scoped define.
        *
        * @param    inDefine
        *           <code>true</code> if processing a define.
        */
       public void setInDefine(boolean inDefine)
       {
          this.inDefine = inDefine;
       }

       /**
        * Gets the flag that identifies if we are currently processing a global or scoped define.
        *
        * @return   <code>true</code> if processing a define.
        */
       public boolean isInDefine()
       {
          return inDefine;
       }

       /**
        * Get the value of last checked IF condition.
        *
        * @return   The state of the {@link #ifCondValue} flag.
        */
       public boolean isIfCondValue()
       {
          return ifCondValue;
       }

       /**
        * Set the value of last checked IF condition.
        *
        * @param   val
        *          The result of the last checked IF condition.
        */
       public void setIfCondValue(boolean val)
       {
          this.ifCondValue = val;
       }

       /**
        * Get the value of the {@link #inSkipBlock} flag.
        *
        * @return   The state of the {@link #inSkipBlock} flag.
        */
       public boolean isInSkipBlock()
       {
          return inSkipBlock;
       }

       /**
        * Set the value of the {@link #inSkipBlock} flag.
        *
        * @param   val
        *          The value.
        */
       public void setInSkipBlock(boolean val)
       {
          this.inSkipBlock = val;
       }

       /**
        * Sets the flag to defer marker processing. When deferred, markers are
        * to be queued instead of pushed back on the input stream.
        *
        * @param    deferMarkers
        *           <code>true</code> to defer marker processing.
        */
       public void setDeferMarkers(boolean deferMarkers)
       {
          this.deferMarkers = deferMarkers;
       }

       /**
        * Gets the flag to defer marker processing. When deferred, markers are
        * to be queued instead of pushed back on the input stream.
        *
        * @return   <code>true</code> to defer marker processing.
        */
       public boolean isDeferMarkers()
       {
          return deferMarkers;
       }

       /**
        * Process any deferred (queued) markers and force the hints to be
        * generated.
        */
       public void clearDeferred()
       {
          try
          {
             filter(new StringBuffer(getMarkers()), false);
          }

          catch (IOException ioe)
          {
             // TODO: output this as an error?
          }
       }

       /**
        * Pushes back some string data into the input reader.
        *
        * @param  data
        *         sequence of characters to be put back into stream
        */
       public void pushBack(String data)
       {
          ins.unread(data.toCharArray());
       }

       /**
        * Pushes a character back into the input reader.
        *
        * @param  c
        *         character to be put back into stream
        */
       public void pushBack(char c)
       {
          ins.unread(c);
       }

       /**
        * Sets the output line number.
        *
        * @param  lineNo
        *         new line number to be set
        */
       public void setLine(int lineNo)
       {
          this.lineNo = lineNo;
       }

       /**
        * Sets the output column number.
        *
        * @param  colNo
        *         new column number to be set
        */
       public void setColumn(int colNo)
       {
          this.colNo = colNo;
       }

       /**
        * Gets the current output line number.
        *
        * @return current line number on the output, starting from 1
        */
       public int getLine()
       {
          return lineNo;
       }

       /**
        * Gets the current output column number.
        * The column number is where the next printed character would appear.
        *
        * @return current column number on the output, starting from 1
        */
       public int getColumn()
       {
          return colNo;
       }

       /**
        * Prints the filtered message to the output stream.
        * Filtering is required to detect, handle and remove markers that
        * the ClearStream could have inserted into the input stream.
        * <p>
        * Markers are sequences of characters that start and end with the marker
        * character, which is configurable through an option.
        *
        * @param  message
        *         text to be put on output stream this Environment prints to.
        * @throws IOException
        *         when an unclosed marker is detected in the text
        */
       public void print(String message)
       throws IOException
       {
          print(new StringBuffer(message));
       }

       /**
        * Prints the filtered message to the output stream.
        * Filtering is required to detect, handle and remove markers that
        * the ClearStream could have inserted into the input stream.
        * <p>
        * Markers are sequences of characters that start and end with the marker
        * character, which is configurable through an option.
        *
        * @param  message
        *         text to be put on output stream this Environment prints to.
        * @throws IOException
        *         when an unclosed marker is detected in the text
        */
       public void print(StringBuffer message)
       throws IOException
       {
          String result = filter(message, true);
          ous.write(result);
       }

       /**
        * Returns the error output stream.
        *
        * @return   Stream used for error messages.
        */
       public PrintWriter getErr()
       {
          return err;
       }

       /**
        * Returns the warning output stream.
        *
        * @return   The stream used for warning messages.
        */
       public PrintWriter getWarn()
       {
          return warn;
       }

       /**
        * Prints an error message without any formatting.
        *
        * @param  message
        *         text to be put on error stream literally
        */
       public void eprint(String message)
       {
          if (getErrorCounter() == 0)
          {
             String toplvl = ((FileScope)sym.getScopeAt(-1)).getFileName();
             err.println(toplvl);
          }

          err.println(message);
          incrErrorCounter();
       }

       /**
        * Formats and prints an error message.
        *
        * @param  message
        *         text to be edited and put on error stream
        * @param  lineno
        *         line number to be inserted into the message
        */
       public void eprint(String message, int lineno)
       {
          eprint(message, lineno, -1);
       }

       /**
        * Formats and prints an error message.
        *
        * @param  message
        *         text to be edited and put on error stream
        * @param  lineno
        *         line number to be inserted into the message
        * @param  colno
        *         column number to be inserted into the message
        */
       public void eprint(String message, int lineno, int colno)
       {
          String filename = ((FileScope)sym.getScope()).getFileName();

          String col = (colno == -1) ? "" : ":" + Integer.toString(colno);

          eprint("Error in "              +
                 filename                 +
                 "#"                      +
                 Integer.toString(lineno) +
                 col                      +
                 " ==> "                  +
                 message);
       }

       /**
        * Formats and prints a warning message.
        *
        * @param  message
        *         text to be edited and put on error stream
        * @param  line
        *         line number to be inserted into the message
        * @param  col
        *         column number to be inserted into the message
        */
       public void wprint(String message, int line, int col)
       {
          String filename = ((FileScope)sym.getScope()).getFileName();
          String result   = String.format("Warning [%s line %d, col %d]: %s",
                                          filename,
                                          line,
                                          col,
                                          message);
          if (getWarningCounter() == 0)
          {
             String toplvl = ((FileScope)sym.getScopeAt(-1)).getFileName();
             warn.println(toplvl);
          }

          warn.println(result);
          incrWarningCounter();
       }

       /**
        * Get the warning counter.  This is stored in the top-level scope.
        *
        * @return   The current warning counter.
        */
       public int getWarningCounter()
       {
          return (rootEnv != null) ? rootEnv.warnings : warnings;
       }

       /**
        * Edits and prints a compile-time message. These kind of messages do not increment the
        * internal error count. Used for the {@code &amp;MESSAGE} preprocessor statement.
        *
        * @param    message
        *           The text to be edited and put on error stream.
        * @param    lineno
        *           The line number to be inserted into the message.
        */
       public void mprint(String message, int lineno)
       {
          String filename = ((FileScope)sym.getScope()).getFile().getName();

          err.println("Message from " +
                      filename        +
                      "#"             +
                      lineno          +
                      " ==> "         +
                      message);
       }

       /**
        * Returns scoped symbol dictionary.
        *
        * @return scoped symbol dictionary this Environment is using
        */
       public ScopedSymbolDictionary getSym()
       {
          return sym;
       }

       /**
        * Returns options object.
        *
        * @return preprocessor options this Environment is using
        */
       public Options getOpt()
       {
          return opt;
       }

       /**
        * Get the error counter.  This is stored in the top-level scope.
        *
        * @return   The current error counter.
        */
       public int getErrorCounter()
       {
          return (rootEnv != null) ? rootEnv.errors : errors;
       }

       /**
        * Attaches the specified arguments Map to this environment.
        *
        * @param  args
        *         a Map that contains the results of arguments parsing
        */
       public void setMap(Map args)
       {
          this.args = args;
       }

       /**
        * Returns the Map attached to this environment.
        *
        * @return
        *         a Map that contains the results of arguments parsing
        */
       public Map getMap()
       {
          return args;
       }

       /**
        * Attaches a hints list to this environment.
        *
        * @param hints
        *        the hints object
        */
       public void setHints(Hints hints)
       {
          this.hints = hints;
       }

       /**
        * Returns the hints list attached to this environment.
        *
        * @return   Current hints object.
        */
       public Hints getHints()
       {
          return hints;
       }

       /**
        * Buffers the markers for deferred insertion.
        *
        * @param    prefix
        *           prefix of the marker.
        * @param    index
        *           marker's index.
        */
       public void queueMarker(String prefix, int index)
       {
          refMarks.append(marker);
          refMarks.append(prefix);
          refMarks.append(Integer.toString(index));
          refMarks.append(marker);
          refMarks.append(" ");
       }

       /**
        * Buffers the markers for deferred insertion.
        *
        * @param    marker
        *           the whole marker as a string.
        */
       public void queueMarker(String marker)
       {
          refMarks.append(marker);
       }

       /**
        * Gets all buffered reference markers as a single string.
        *
        * @return  all buffered markers
        */
       public String getMarkers()
       {
          if (refMarks == null)
             return "";
          else
          {
             String ret = refMarks.toString();
             refMarks.setLength(0);
             return ret;
          }
       }

       /**
        * Scans the message for markers and newlines, handles and removes markers
        * and adjusts current line and column on the output stream.
        * <p>
        * Markers are followed by a space separating them from the text. Those
        * spaces may come either as part of the same token (in comments and
        * strings) or as part of a separate WS token. In any case, the separating
        * space has to be removed.
        *
        * @param  buf
        *         text about to be put on output stream.
        * @param  adjust
        *         <code>true</code> if line and column numbers have to be adjusted
        * @return filtered text without markers
        * @throws IOException
        *         when an unclosed marker is detected in the text
        */
       public String filter(StringBuffer buf, boolean adjust)
       throws IOException
       {
          if (buf.length() == 0)
             return "";

          // if space removal is pending, do it first
          if (pendingSpaceRemoval)
          {
             pendingSpaceRemoval = false;
             if (buf.charAt(0) != ' ')
                throw new IOException("lost separating space: <" +
                                      buf.toString() + ">");
             buf.delete(0, 1);
          }

          // detect and handle markers
          String mark = Character.toString(marker);
          int mpos = buf.indexOf(mark);
          int mend = 0;
          String label = null;

          while (mpos >= 0)
          {
             mend = buf.indexOf(mark, mpos + 1);
             if (mend == -1)
                throw new IOException("unclosed marker on the output");
             label = buf.substring(mpos + 1, mend);
             handleMarker(label);
             // detect the separating space
             if (buf.length() > mend + 1)
             {
                if (buf.charAt(mend + 1) != ' ')
                   throw new IOException("lost separating space: <" +
                                         buf.toString() + ">");
                mend++;      // delete also the separating space
             }
             else
             {
                pendingSpaceRemoval = true;
             }
             buf.delete(mpos, mend + 1);
             mpos = buf.indexOf(mark);
          }

          // create the output string
          String message = buf.toString();

          // adjust line and column numbers depending on the number of newlines
          if (adjust)
          {
             int nls = StringHelper.countChar(message, '\n');
             if (nls == 0)
                colNo += message.length();
             else
             {
                lineNo += nls;
                colNo = message.length() - message.lastIndexOf('\n');
             }
          }

          return message;
       }

       /**
        * Increment the warning counter.  This is stored in the top-level scope.
        */
       private void incrWarningCounter()
       {
          if (rootEnv != null)
          {
             rootEnv.warnings++;
          }
          else
          {
             warnings++;
          }
       }

       /**
        * Increment the error counter.  This is stored in the top-level scope.
        */
       private void incrErrorCounter()
       {
          if (rootEnv != null)
          {
             rootEnv.errors++;
          }
          else
          {
             errors++;
          }
       }

       /**
        * Identifies the marker and adds the corresponding element into the hints
        * tree.
        * <p>
        * The contents of the marker can be:
        * <ul>
        *   <li>include file hint "i" + index
        *   <li>argument hint     "a" + index
        *   <li>reference hint    "r" + index
        *   <li>end of include hint - empty
        * </ul>
        *
        * @param  label
        *         contents of the marker
        */
       private void handleMarker(String label)
       {
          int index = 0;
          List  list  = hints.getHintsList();
          Deque stack = hints.getHintsStack();

          if (label.length() > 0)
          {
             index = Integer.parseInt(label.substring(1));

             if (label.startsWith("i"))
             {
                IncludeHint ih = (IncludeHint)list.get(index);
                ih.setStartLine(lineNo);
                ih.setStartColumn(colNo);
                stack.push(ih);
                hints.createAndPushIncludeHint(ih);
             }

             if (label.startsWith("a"))
             {
                ArgumentHint ah = (ArgumentHint)list.get(index);
                hints.createArgumentHint(ah);
             }

             if (label.startsWith("r"))
             {
                ReferenceHint rh = (ReferenceHint)list.get(index);
                rh.setLine(lineNo);
                rh.setColumn(colNo);
                hints.createReferenceHint(rh);
             }
          }
          else
          {
             if (stack.size() > 0)
             {
                IncludeHint ih = (IncludeHint)stack.pop();
                ih.setEndLine(lineNo);
                ih.setEndColumn(colNo);
                hints.popAndCompleteIncludeHint(ih);
             }
             else
             {
                String defs = refMarks == null ? "" : refMarks.toString().replace(opt.getMarker(), '^');
                wprint(String.format("UNBALANCED EOI MARKER (deferred = '%s')\n", defs), lineNo, colNo);
             }
          }
       }
    }