Environment.java

/*
** Module   : Environment.java
** Abstract : Encapsulates the shared environment for parsers, lexers and drivers.
**
** Copyright (c) 2004-2020, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ---------------------------------Description-----------------------------------
** 001 NVS 20041109   @18765 Created. Environment class is a collection
**                           of references and methods giving access to
**                           all objects shared between the ANTLR
**                           generated code and the manually written code.
** 002 NVS 20050207   @19629 Added support for dynamic arguments parsing.
**                           Environment class now provides a private
**                           member that may point to a Map with parsed 
**                           arguments, and setMap and getMap methods.
** 003 NVS 20050413   @20689 Inserted a TO DO marker around System.exit().
** 004 NVS 20050510   @21147 System.exit() replaced with the IOException
**                           in case there is no room in the pushback
**                           buffer.
**                           Added line and column information tracking
**                           for the preprocessor output stream. Added
**                           methods to set and get line and column
**                           numbers. Added markers support code.
** 005 NVS 20050517   @21214 handleMarker throws an exception when used as
**                           -lexonly. Made that code conditional.
**                           Fixed marker handling bug. Markers now
**                           include a trailing space. See ##21210-21212
**                           for details.
** 006 NVS 20050518   @21237 Simplified the marker handling.
** 007 NVS 20050526   @21311 javadoc fixes
** 008 GES 20071204   @36193 Added a flag to disable the expansions done
**                           by the braces parser. Minor refactoring for
**                           code reduction. Added defer marker processing
**                           flag to help control braces expansion. Added
**                           a helper for obtaining lookahead chars.
** 009 GES 20080308   @37647 Switched to using character readers and
**                           writers instead of byte streams. This handles
**                           I18N needs.
** 010 GES 20081010   @40334 Removed unnecessary throws clauses.
** 011 GES 20090424   @41949 Import change.
** 012 GES 20110628          Lexer multiplexing/switching is no longer needed.
**                           Fixed bug in lookahead() where wrong text is seen
**                           because the syncConsume() was not called in some
**                           cases.
** 013 GES 20110901          Added warning message support.
** 014 EVL 20160223          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
** 015 CA  20170825          Collect all preprocessor symbols via addConstantDefinitions.
** 016 CA  20170831          In ASTMT, the lexer must not set 'inStatement' set unless what we 
**                           are matching really is a statement.
**                           If a AELSE or AELIF is matched with IF condition true (i.e.THEN 
**                           branch is executed), disable the brace expansio, as the body will
**                           not execute.
** 017 GES 20180926          Added inDefine flag.
** 018 GES 20200306          Minor code cleanup and dead code removal.
** 019 GES 20200625          Created common helper to calculate push vs queue for markers. Added code to
**                           detect unbalanced end of include file markers.
**     GES 20200629          Made the unbalanced eoi into a warning (an error is too much since this
**                           condition can be ignored).
** 020 CA  20201015          Replaced java.util.Stack with ArrayDeque (as synchronization is not required).
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.preproc;

import com.goldencode.util.*;
import com.goldencode.p2j.util.*; 
import java.io.*;
import java.util.*;
import javax.xml.parsers.ParserConfigurationException;

import antlr.collections.AST;
import antlr.collections.impl.*;
import antlr.debug.misc.*;
import antlr.*;

/**
 * Encapsulates the shared environment for parsers, lexers and drivers.
 * There must be an instance of the class shared between parsers and lexers
 * if they are to work on the same input and output. However, multiple 
 * environments can be instantiated to conduct multiple unrelated "threads" 
 * of work with their independent parsers and lexers, if necessary.
 * <p>
 * An instance of the Environment is given as a parameter to constructors
 * of parsers and lexers. This is how they can access this information
 * later in grammar actions.
 * <p>
 * This class also provides a number of useful shortcuts for method calls.
 * These shortcuts reduce the verbosity of the grammar actions and make
 * the grammar files more readable.
 * <p>
 * The latest revision of this class maintains the line and column information
 * for the output stream. These can be read and set. This functionality is
 * required for tracking the portions of the output generated by a specific
 * include file.
 */
public class Environment
{
   /** Parent node or <code>null</code>. */
   private Environment parent = null;

   /** Top level (parent of all parents) or <code>null</code>. */
   private Environment rootEnv = null;
   
   /** Type of the environment - file (true) or reference (false). */
   private boolean file = true;

   /** ANTLR lexer shared input state object */
   private LexerSharedInputState lsi = null;

   /** ANTLR parser shared input state object */
   private ParserSharedInputState psi = null;

   /** reference to a lexer for calling <code>newline()</code> etc */
   private CharScanner lexer = null;

   /** reference to a parser for calling <code>setFileName()</code> */
   private Parser parser = null;

   /** input stream with a large pushback buffer */
   private ClearStream ins = null;

   /** Stream for preprocessor output. */
   private Writer ous = null;

   /** current line number for the preprocessor output */
   private int lineNo = 1;

   /** current column number for the preprocessor output */
   private int colNo = 1;

   /** Stream for error messages. */
   private PrintWriter err = null;

   /** Stream for warning messages. */
   private PrintWriter warn = null;
   
   /** Scoped symbol dictionary. */
   private ScopedSymbolDictionary sym = null;

   /** preprocessor options */
   private Options opt = null;

   /** counter for the preprocessor errors */
   private int errors = 0;
   
   /** Warnings counter. */
   private int warnings = 0;

   /** reference to a Map used to parse arguments only */
   private Map args = null;

   /** Configurable marker character.
    *  Markers are inserted into the input stream to indicate boundaries of
    *  included files, references etc.
    */
   private char marker = '\001';

   /** reference to Hints */
   private Hints hints = null;

   /** buffered reference hint markers */
   private StringBuffer refMarks = null;

   /** <code>true</code> if in a preprocessor statement. */
   private boolean inStat = false;

   /** <code>true</code> if space removal is pending. */
   private boolean pendingSpaceRemoval = false;

   /** Disable/enable braces parser expansions. */
   private boolean expandBraces = true;

   /** Flag to identify when we are processing a global or scoped definition. */
   private boolean inDefine = false;

   /** Queue markers instead of pushing them back on the input stream. */
   private boolean deferMarkers = false;
   
   /** The value of the last executed IF or ELSEIF statement. */
   private boolean ifCondValue = false;
   
   /** Flag indicating the parser is operating a skipBlock. */
   private boolean inSkipBlock = false;
   
   /**
    * Construct an instance.
    *
    * @param    parent
    *           Parent Environment object or <code>null</code>.
    * @param    ins
    *           Input stream, an instance of ClearStream providing
    *           input for all lexers using this environment.
    * @param    ous
    *           Stream for preprocessor output.
    * @param    err
    *           Stream for preprocessor error messages.
    * @param    warn
    *           Stream for preprocessor warning messages.
    * @param    sym
    *           Scoped symbol dictionary.
    * @param    opt
    *           Configuration options.
    *
    * @throws   ParserConfigurationException
    *           Forwarded from XmlHelper.
    * @throws   IOException
    *           Forwarded from XmlHelper.
    */
   public Environment(Environment            parent,
                      ClearStream            ins,
                      Writer                 ous,
                      Writer                 err,
                      Writer                 warn,
                      ScopedSymbolDictionary sym,
                      Options                opt)
   throws ParserConfigurationException,
          IOException
   {
      this.parent = parent;
      this.ins    = ins;
      this.ous    = ous;
      this.sym    = sym;
      this.opt    = opt;
      
      if (err != null)
      {
         this.err = (err instanceof PrintWriter) ? (PrintWriter) err 
                                                 : new PrintWriter(err);
         this.warn = this.err;
      }
      
      if (warn != null)
      {
         this.warn = (warn instanceof PrintWriter) ? (PrintWriter) warn 
                                                   : new PrintWriter(warn);
      }
      
      if (parent != null)
      {
         // propagate the parents braces expansion setting if there is a
         // parent
         expandBraces = parent.expandBraces;
         
         Environment next = parent;
         
         while (next != null)
         {
            rootEnv = next;
            next = next.parent;
         }
      }
      
      marker = opt.getMarker();
      
      if (opt.getHintsFile() != null)
      {
         hints = new Hints(opt.getHintsFile());
         refMarks = new StringBuffer();
      }
   }
   
   /**
    * Add all preprocessor level symbol definitions into the hints, for the current scope.  
    */
   public void addConstantDefinitions()
   {
      if (hints == null)
      {
         return;
      }
      
      Map<?, ?> dict = sym.getDictionaryAtScope(0, false);
      
      for (Object key : dict.keySet())
      {
         if (key instanceof String)
         {
            Object val = dict.get(key);
            
            if (val instanceof Symbol)
            {
               Symbol b = (Symbol) val;
               
               hints.createConstantSymbol((String) key, b);
            }
         }
      }
   }

   /**
    * Sets the environment type.
    *
    * @param    file
    *           <code>true</code> if this environment is for a file. 
    *           Otherwise is a reference.
    */
   public void setFile(boolean file)
   {
      this.file = file;
   }

   /**
    * Gets the environment type.
    *
    * @return   <code>true</code> if this environment is for a file.
    *           Otherwise is a reference.
    */
   public boolean isFile()
   {
      return file;
   }

   /**
    * Gets the parent environment.
    *
    * @return parent Environment object or <code>null</code>.
    */
   public Environment getParent()
   {
      return parent;
   }
   
   /**
    * Report if markers should be pushed or queued.  Pushing writes the markers into the stream immediately
    * where queuing defers the processing of markers until a safe time.
    *
    * @return   {@code true} if markers should be pushed.
    */
   public boolean isPush()
   {
      return isFile() && !isInStatement() && !isDeferMarkers();
   }

   /**
    * Returns the LexerSharedInputState object.
    *
    * @return ANTLR LexerSharedInputState associated with
    *         this Environment
    */
   public LexerSharedInputState getLsi()
   {
      return lsi;
   }

   /**
    * Sets the LexerSharedInputState object.
    *
    * @param  lsi
    *         ANTLR LexerSharedInputState to be associated with
    *         this Environment
    */
   public void setLsi(LexerSharedInputState lsi)
   {
      this.lsi = lsi;
   }

   /**
    * Returns the ParserSharedInputState object.
    *
    * @return ANTLR ParserSharedInputState associated with
    *         this Environment
    */
   public ParserSharedInputState getPsi()
   {
      return psi;
   }

   /**
    * Sets the ParserSharedInputState object.
    *
    * @param  psi
    *         ANTLR ParserSharedInputState to be associated with
    *         this Environment
    */
   public void setPsi(ParserSharedInputState psi)
   {
      this.psi = psi;
   }

   /**                                      
    * Obtain the lookahead of the given or current lexer as a string. If the
    * result is the single character <code>EOF</code> or if there is no
    * lookahead, the empty string is returned.
    *
    * @param    lexer
    *           The lexer from which to obtain the lookahead or 
    *           <code>null</code> to use the currently defined lexer within
    *           this instance.
    *
    * @return   Buffered characters already read from the stream.
    */
   public String lookahead(CharScanner lexer)
   {
      if (lexer == null)
      {
         lexer = this.lexer;
      }
      
      InputBuffer inBuf = lexer.getInputBuffer();
      
      // we don't really need to "mark", but it is harmless and calls
      // syncConsume() which is VERY important because otherwise the LA
      // chars buffer will still have unconsumed content
      inBuf.mark();
      
      // it is now safe to get the lookahead
      String buffered = inBuf.getLAChars();
      
      // clear the "mark"
      inBuf.commit();
      
      if (buffered == null)
      {
         buffered = "";
      }
      else if (buffered.length() == 1)
      {
         // check for EOF
         if (buffered.charAt(0) == 0xFFFF)
         {
            // don't push EOF back 
            buffered = "";
         }
      }
      
      return buffered;
   }
   
   /**
    * Attaches the specified lexer to this environment.
    *
    * @param  lexer
    *         ANTLR CharScanner lexer to be attached to
    *         this Environment
    */
   public void setLex(CharScanner lexer)
   {
      this.lexer = lexer;
   }

   /**
    * Returns the lexer attached to this environment.
    *
    * @return
    *         ANTLR CharScanner lexer attached to this Environment
    */
   public CharScanner getLex()
   {
      return lexer;
   }

   /**
    * Attaches the specified parser to this environment.
    *
    * @param  parser
    *         ANTLR Parser to be attached to this Environment
    */
   public void setPar(Parser parser)
   {
      this.parser = parser;
   }

   /**
    * Returns the parser attached to this environment.
    *
    * @return
    *         ANTLR Parser attached to this Environment
    */
   public Parser getPar()
   {
      return parser;
   }

   /**
    * Returns the input source.
    *
    * @return   The input source for this environment.
    */
   public Reader getIns()
   {
      return ins;
   }

   /**
    * Sets the inComment state.
    *
    * @param  inComment
    *         new state of the flag
    */
   public void setInComment(boolean inComment)
   {
      ins.setInComment(inComment);
   }

   /**
    * Sets the inString state.
    *
    * @param  inString
    *         new state of the flag
    */
   public void setInString(boolean inString)
   {
      ins.setInString(inString);
   }

   /**
    * Detects if the processing is inside a string literal.
    *
    * @return   <code>true</code> if the current processing is inside a
    *           string literal.
    */
   public boolean isInString()
   {
      return ins.isInString();
   }
   
   /**
    * Sets the inStatement state.
    *
    * @param  inStat
    *         new state of the flag
    */
   public void setInStatement(boolean inStat)
   {
      this.inStat = inStat;
   }

   /**
    * Gets the inStatement state.
    *
    * @return <code>true</code> if in statement flag is set
    */
   public boolean isInStatement()
   {
      return (rootEnv != null) ? rootEnv.inStat : inStat;
   }

   /**
    * Sets the flag to enable or disable the braces parser's expansion of
    * braces.
    *
    * @param    expandBraces
    *           <code>true</code> to enable.
    */
   public void setExpandBraces(boolean expandBraces)
   {
      this.expandBraces = expandBraces;
   }

   /**
    * Gets the flag that enables or disables the braces parser's expansion of
    * braces.
    *
    * @return   <code>true</code> if braces expansion is enabled.
    */
   public boolean isExpandBraces()
   {
      return expandBraces;
   }

   /**
    * Sets the flag that identifies if we are currently processing a global or scoped define.
    *
    * @param    inDefine
    *           <code>true</code> if processing a define.
    */
   public void setInDefine(boolean inDefine)
   {
      this.inDefine = inDefine;
   }

   /**
    * Gets the flag that identifies if we are currently processing a global or scoped define.
    *
    * @return   <code>true</code> if processing a define.
    */
   public boolean isInDefine()
   {
      return inDefine;
   }

   /**
    * Get the value of last checked IF condition.
    * 
    * @return   The state of the {@link #ifCondValue} flag. 
    */
   public boolean isIfCondValue()
   {
      return ifCondValue;
   }
   
   /**
    * Set the value of last checked IF condition.
    * 
    * @param   val
    *          The result of the last checked IF condition. 
    */
   public void setIfCondValue(boolean val)
   {
      this.ifCondValue = val;
   }

   /**
    * Get the value of the {@link #inSkipBlock} flag.
    * 
    * @return   The state of the {@link #inSkipBlock} flag. 
    */
   public boolean isInSkipBlock()
   {
      return inSkipBlock;
   }
   
   /**
    * Set the value of the {@link #inSkipBlock} flag.
    * 
    * @param   val
    *          The value. 
    */
   public void setInSkipBlock(boolean val)
   {
      this.inSkipBlock = val;
   }

   /**
    * Sets the flag to defer marker processing. When deferred, markers are
    * to be queued instead of pushed back on the input stream.
    *
    * @param    deferMarkers
    *           <code>true</code> to defer marker processing.
    */
   public void setDeferMarkers(boolean deferMarkers)
   {
      this.deferMarkers = deferMarkers;
   }

   /**
    * Gets the flag to defer marker processing. When deferred, markers are
    * to be queued instead of pushed back on the input stream.
    *
    * @return   <code>true</code> to defer marker processing.
    */
   public boolean isDeferMarkers()
   {
      return deferMarkers;
   }
   
   /**
    * Process any deferred (queued) markers and force the hints to be
    * generated.
    */
   public void clearDeferred()
   {
      try
      {
         filter(new StringBuffer(getMarkers()), false);
      }
      
      catch (IOException ioe)
      {
         // TODO: output this as an error?
      }
   }

   /**
    * Pushes back some string data into the input reader.
    *
    * @param  data
    *         sequence of characters to be put back into stream
    */
   public void pushBack(String data)
   {
      ins.unread(data.toCharArray());
   }

   /**
    * Pushes a character back into the input reader.
    *
    * @param  c
    *         character to be put back into stream
    */
   public void pushBack(char c)
   {
      ins.unread(c);
   }

   /**
    * Sets the output line number.
    *
    * @param  lineNo
    *         new line number to be set
    */
   public void setLine(int lineNo)
   {
      this.lineNo = lineNo;
   }

   /**
    * Sets the output column number.
    *
    * @param  colNo
    *         new column number to be set
    */
   public void setColumn(int colNo)
   {
      this.colNo = colNo;
   }

   /**
    * Gets the current output line number.
    *
    * @return current line number on the output, starting from 1
    */
   public int getLine()
   {
      return lineNo;
   }

   /**
    * Gets the current output column number.
    * The column number is where the next printed character would appear.
    *
    * @return current column number on the output, starting from 1
    */
   public int getColumn()
   {
      return colNo;
   }

   /**
    * Prints the filtered message to the output stream.
    * Filtering is required to detect, handle and remove markers that
    * the ClearStream could have inserted into the input stream.
    * <p>
    * Markers are sequences of characters that start and end with the marker
    * character, which is configurable through an option.
    *
    * @param  message
    *         text to be put on output stream this Environment prints to.
    * @throws IOException
    *         when an unclosed marker is detected in the text
    */
   public void print(String message)
   throws IOException
   {
      print(new StringBuffer(message));
   }

   /**
    * Prints the filtered message to the output stream.
    * Filtering is required to detect, handle and remove markers that
    * the ClearStream could have inserted into the input stream.
    * <p>
    * Markers are sequences of characters that start and end with the marker
    * character, which is configurable through an option.
    *
    * @param  message
    *         text to be put on output stream this Environment prints to.
    * @throws IOException
    *         when an unclosed marker is detected in the text
    */
   public void print(StringBuffer message)
   throws IOException
   {
      String result = filter(message, true);
      ous.write(result);
   }

   /**
    * Returns the error output stream.
    *
    * @return   Stream used for error messages.
    */
   public PrintWriter getErr()
   {
      return err;
   }

   /**
    * Returns the warning output stream.
    *
    * @return   The stream used for warning messages.
    */
   public PrintWriter getWarn()
   {
      return warn;
   }

   /**
    * Prints an error message without any formatting.
    *
    * @param  message
    *         text to be put on error stream literally
    */
   public void eprint(String message)
   {
      if (getErrorCounter() == 0)
      {
         String toplvl = ((FileScope)sym.getScopeAt(-1)).getFileName();
         err.println(toplvl);
      }
      
      err.println(message);
      incrErrorCounter();
   }

   /**
    * Formats and prints an error message.
    *
    * @param  message
    *         text to be edited and put on error stream
    * @param  lineno
    *         line number to be inserted into the message
    */
   public void eprint(String message, int lineno)
   {
      eprint(message, lineno, -1);
   }

   /**
    * Formats and prints an error message.
    *
    * @param  message
    *         text to be edited and put on error stream
    * @param  lineno
    *         line number to be inserted into the message
    * @param  colno
    *         column number to be inserted into the message
    */
   public void eprint(String message, int lineno, int colno)
   {
      String filename = ((FileScope)sym.getScope()).getFileName();
      
      String col = (colno == -1) ? "" : ":" + Integer.toString(colno);
      
      eprint("Error in "              +
             filename                 +
             "#"                      +
             Integer.toString(lineno) +
             col                      +
             " ==> "                  +
             message);
   }

   /**
    * Formats and prints a warning message.
    *
    * @param  message
    *         text to be edited and put on error stream
    * @param  line
    *         line number to be inserted into the message
    * @param  col
    *         column number to be inserted into the message
    */
   public void wprint(String message, int line, int col)
   {
      String filename = ((FileScope)sym.getScope()).getFileName();
      String result   = String.format("Warning [%s line %d, col %d]: %s",
                                      filename,
                                      line,
                                      col,
                                      message);
      if (getWarningCounter() == 0)
      {
         String toplvl = ((FileScope)sym.getScopeAt(-1)).getFileName();
         warn.println(toplvl);
      }
      
      warn.println(result);
      incrWarningCounter();
   }
   
   /**
    * Get the warning counter.  This is stored in the top-level scope.
    *
    * @return   The current warning counter.
    */
   public int getWarningCounter()
   {
      return (rootEnv != null) ? rootEnv.warnings : warnings;
   }

   /**
    * Edits and prints a compile-time message. These kind of messages do not increment the
    * internal error count. Used for the {@code &amp;MESSAGE} preprocessor statement.
    *
    * @param    message
    *           The text to be edited and put on error stream.
    * @param    lineno
    *           The line number to be inserted into the message.
    */
   public void mprint(String message, int lineno)
   {
      String filename = ((FileScope)sym.getScope()).getFile().getName();
      
      err.println("Message from " + 
                  filename        +
                  "#"             +
                  lineno          +
                  " ==> "         +
                  message);
   }

   /**
    * Returns scoped symbol dictionary.
    *
    * @return scoped symbol dictionary this Environment is using
    */
   public ScopedSymbolDictionary getSym()
   {
      return sym;
   }

   /**
    * Returns options object.
    *
    * @return preprocessor options this Environment is using
    */
   public Options getOpt()
   {
      return opt;
   }

   /**
    * Get the error counter.  This is stored in the top-level scope.
    *
    * @return   The current error counter.
    */
   public int getErrorCounter()
   {
      return (rootEnv != null) ? rootEnv.errors : errors;
   }

   /**
    * Attaches the specified arguments Map to this environment.
    *
    * @param  args
    *         a Map that contains the results of arguments parsing
    */
   public void setMap(Map args)
   {
      this.args = args;
   }

   /**
    * Returns the Map attached to this environment.
    *
    * @return
    *         a Map that contains the results of arguments parsing
    */
   public Map getMap()
   {
      return args;
   }

   /**
    * Attaches a hints list to this environment.
    *
    * @param hints
    *        the hints object
    */
   public void setHints(Hints hints)
   {
      this.hints = hints;
   }

   /**
    * Returns the hints list attached to this environment.
    *
    * @return   Current hints object.
    */
   public Hints getHints()
   {
      return hints;
   }

   /**
    * Buffers the markers for deferred insertion.
    *
    * @param    prefix
    *           prefix of the marker.
    * @param    index
    *           marker's index.
    */
   public void queueMarker(String prefix, int index)
   {
      refMarks.append(marker);
      refMarks.append(prefix);
      refMarks.append(Integer.toString(index));
      refMarks.append(marker);
      refMarks.append(" ");
   }

   /**
    * Buffers the markers for deferred insertion.
    *
    * @param    marker
    *           the whole marker as a string.
    */
   public void queueMarker(String marker)
   {
      refMarks.append(marker);
   }

   /**
    * Gets all buffered reference markers as a single string.
    *
    * @return  all buffered markers
    */
   public String getMarkers()
   {
      if (refMarks == null)
         return "";
      else
      {
         String ret = refMarks.toString();
         refMarks.setLength(0);
         return ret;
      }
   }

   /**
    * Scans the message for markers and newlines, handles and removes markers
    * and adjusts current line and column on the output stream.
    * <p>
    * Markers are followed by a space separating them from the text. Those
    * spaces may come either as part of the same token (in comments and 
    * strings) or as part of a separate WS token. In any case, the separating
    * space has to be removed.
    *
    * @param  buf
    *         text about to be put on output stream.
    * @param  adjust
    *         <code>true</code> if line and column numbers have to be adjusted
    * @return filtered text without markers
    * @throws IOException
    *         when an unclosed marker is detected in the text
    */
   public String filter(StringBuffer buf, boolean adjust)
   throws IOException
   {
      if (buf.length() == 0)
         return "";

      // if space removal is pending, do it first
      if (pendingSpaceRemoval)
      {
         pendingSpaceRemoval = false;
         if (buf.charAt(0) != ' ')
            throw new IOException("lost separating space: <" + 
                                  buf.toString() + ">");
         buf.delete(0, 1);
      }

      // detect and handle markers
      String mark = Character.toString(marker);
      int mpos = buf.indexOf(mark);
      int mend = 0;
      String label = null;

      while (mpos >= 0)
      {
         mend = buf.indexOf(mark, mpos + 1);
         if (mend == -1)
            throw new IOException("unclosed marker on the output");
         label = buf.substring(mpos + 1, mend);
         handleMarker(label);
         // detect the separating space
         if (buf.length() > mend + 1)
         {
            if (buf.charAt(mend + 1) != ' ')
               throw new IOException("lost separating space: <" + 
                                     buf.toString() + ">");
            mend++;      // delete also the separating space
         }
         else
         {
            pendingSpaceRemoval = true;
         }
         buf.delete(mpos, mend + 1);
         mpos = buf.indexOf(mark);
      }

      // create the output string
      String message = buf.toString();

      // adjust line and column numbers depending on the number of newlines
      if (adjust)
      {
         int nls = StringHelper.countChar(message, '\n');
         if (nls == 0)
            colNo += message.length();
         else
         {
            lineNo += nls;
            colNo = message.length() - message.lastIndexOf('\n');
         }
      }

      return message;
   }

   /**
    * Increment the warning counter.  This is stored in the top-level scope.
    */
   private void incrWarningCounter()
   {
      if (rootEnv != null)
      {
         rootEnv.warnings++;
      }
      else
      {
         warnings++;
      }
   }
   
   /**
    * Increment the error counter.  This is stored in the top-level scope.
    */
   private void incrErrorCounter()
   {
      if (rootEnv != null)
      {
         rootEnv.errors++;
      }
      else
      {
         errors++;
      }
   }
   
   /**
    * Identifies the marker and adds the corresponding element into the hints
    * tree.
    * <p>
    * The contents of the marker can be:
    * <ul>
    *   <li>include file hint "i" + index
    *   <li>argument hint     "a" + index
    *   <li>reference hint    "r" + index
    *   <li>end of include hint - empty
    * </ul>
    *
    * @param  label
    *         contents of the marker
    */
   private void handleMarker(String label)
   {
      int index = 0;
      List  list  = hints.getHintsList();
      Deque stack = hints.getHintsStack();

      if (label.length() > 0)
      {
         index = Integer.parseInt(label.substring(1));

         if (label.startsWith("i"))
         {
            IncludeHint ih = (IncludeHint)list.get(index);
            ih.setStartLine(lineNo);
            ih.setStartColumn(colNo);
            stack.push(ih);
            hints.createAndPushIncludeHint(ih);
         }

         if (label.startsWith("a"))
         {
            ArgumentHint ah = (ArgumentHint)list.get(index);
            hints.createArgumentHint(ah);
         }

         if (label.startsWith("r"))
         {
            ReferenceHint rh = (ReferenceHint)list.get(index);
            rh.setLine(lineNo);
            rh.setColumn(colNo);
            hints.createReferenceHint(rh);
         }
      }
      else
      {
         if (stack.size() > 0)
         {
            IncludeHint ih = (IncludeHint)stack.pop();
            ih.setEndLine(lineNo);
            ih.setEndColumn(colNo);
            hints.popAndCompleteIncludeHint(ih);
         }
         else
         {
            String defs = refMarks == null ? "" : refMarks.toString().replace(opt.getMarker(), '^');
            wprint(String.format("UNBALANCED EOI MARKER (deferred = '%s')\n", defs), lineNo, colNo);
         }
      }
   }
}