# Preprocessor.java

    /*
    ** Module   : Preprocessor.java
    ** Abstract : external interface to the Progress 4GL compatible preprocessor
    **
    ** Copyright (c) 2004-2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ----------------------------------Description-----------------------------------
    ** 001 NVS 20041108   @18767 Created. This is the main entry point to the
    **                           Preprocessor and the command line driver
    **                           that calls it.
    ** 002 NVS 20041221   @19045 Added new optional mode, -lexonly, which runs
    **                           the lexer through the input file and dumps
    **                           the stream of tokens.
    ** 003 NVS 20041222   @19064 Added new static method, stripNL for deleting
    **                           newlines from character array and processing
    **                           CRCR and CRNL escapes.
    ** 004 NVS 20050207   @19628 The Preprocessor now supports a special run
    **                           mode that only parses a string of arguments.
    **                           This support is needed for Progress RUN
    **                           statement where arguments are dynamic. The
    **                           technique resembles that of using the Prepro-
    **                           cessor invocation command line as a virtual
    **                           include file. The includeFile method detects
    **                           the use of a special file name '.', which
    **                           cannot be an include file anyway, as a parse
    **                           only mode.
    ** 005 NVS 20050207   @19633 Default pushback buffer depth increased to
    **                           4096 due to a dynamic RUN statement found in
    **                           one of the source files where 1024 wasn't
    **                           enough.
    ** 006 NVS 20050209   @19731 Fixed a bug in getVariable method. Arguments
    **                           should be taken from the current scope ONLY.
    **                           If there is no definition for an argument in
    **                           the current scope, but the dictionary finds
    **                           an older definition, that one should be
    **                           ignored.
    ** 007 NVS 20050210   @19756 Enhanced debugging features by adding Clear-
    **                           Stream state dump, current token text and
    **                           lookahead buffers and current dictionary
    **                           scope dump, also in case of exception.
    ** 008 NVS 20050211   @19781 Fixed a bug in preprocessor related to the
    **                           top level include file handling. The name of
    **                           the virtual file in the FileScope constructor
    **                           call below changed to "/". This signifies a
    **                           name than NEVER can be a real filename.
    **                           ClearStream checks the name to see if the top
    **                           level scope is for this virtual name and then
    **                           bypasses appending an extra space character,
    **                           which is otherwise required.
    ** 009 NVS 20050413   @20692 Instead of exiting the process, throws the PreprocessorException when
    **                           preprocessing problems are detected.
    ** 010 NVS 20050511   @21151 Parsing of additional options -hints and -marker. Tracking argument usage in
    **                           include files.
    ** 011 NVS 20050516   @21215 Fixed marker handling bug. Markers now include a trailing space. See
    **                           ##21210-21212 for details.
    ** 012 NVS 20050518   @21238 Simplified the marker handling.
    ** 013 GES 20070319   @32503 Simplified the expression evaluation and integrated with the replacement
    **                           version of the expression processor.  Added better debugging output on error.
    ** 014 GES 20070403   @32712 Support new command line option for honoring unix escape sequences (or not).
    **                           Simplified options usage.
    ** 015 GES 20070717   @34576 Minor interface change for built-in vars.
    ** 016 GES 20070816   @34887 Cleanup and simplification of built-in var reference processing. Updated the
    **                           command line interface to support new options (which provide external control
    **                           over BATCH-MODE, OPSYS and WINDOW-SYSTEM built-in vars).
    ** 017 GES 20071120   @36195 Modification to include processing to support deferred text pushback.
    ** 018 GES 20080307   @37651 Minor fixes to include file processing and changes to allow alternate locales
    **                           to work properly. Removed argument string parsing hack that is no longer needed.
    ** 019 GES 20081010   @40335 Fixes for braces pushback issues, empty &IF expressions, output/error stream
    **                           flushing and error handling.
    ** 020 GES 20090424   @41951 Import change.
    ** 021 GES 20090428   @42021 Shifted to use common code for lexer dumping.
    ** 022 ECF 20101001          Added '-case-sensitive' include file name command line option. Modified
    **                           FileScope usage to account for this option.
    ** 023 GES 20110623          Added basepath and fallback list processing. Removed lexer multiplexing and
    **                           switching.
    ** 024 GES 20110901          Added warning message destination support. Changed include file and fallback
    **                           search errors into warnings.
    ** 025 CS  20121114          Implement PROPATH assignment: custom call of EnvironmentOps.setSearchPath to
    **                           differentiate it from runtime call.
    ** 026 AIL 20130128          Update from character to TextOps.
    ** 027 EVL 20160223          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
    ** 028 HC  20160802          Fixed handling of file separators when including files during preprocessing.
    ** 029 HC  20170709          Added source-charset uast preprocessor hint.
    ** 030 CA  20170825          Enhanced to allow collection and reporting of preprocessor constant symbols.
    **                           Only top-level symbols are collected for now.
    ** 031 CA  20180321          Added support for PROCESS-ARCHITECTURE preprocessor var.
    ** 032 CA  20180413          Javadoc fix.
    ** 033 HC  20190227          Fixed hints files not loaded by the preprocessor.
    ** 034 CA  20190416          Fixed stripNL for multiple consecutive NL case.
    ** 035 CA  20190423          Completed fix for H034.
    ** 036 ECF 20190619          Added support for include file reference prefix remapping.
    **     GES 20190929          Fixed implementation of the DEFINED() preproc built-in, it must process the
    **                           contents up to the next RPARENS as simple text without treating them as a
    **                           sub-expression.  See fixupDefined().
    **     RFB 20190930          Minor update to fixupDefined() to blast newline, which was factoring
    **                           into stringify's calculations for where to put the double-quotes.
    **     GES 20191010          Added process-architecture builtin function callback.
    **     GES 20191013          Trim symbols before looking them up in DEFINED().
    ** 037 GES 20200626          Fixed process-architecture expression evaluation and fixed an error message.
    ** 038 GES 20220318          Expression evaluation failures in the 4GL don't abend and should not report as
    **                           an error. These already return false by default, now they report as warnings.
    **     CA  20220418          Each top-file has a local preprocessor variable scope, beside the global scope.
    **     TJD 20220504          Java 11 compatibility minor changes
    ** 039 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
    ** 040 EVL 20250303          Adding method to remove slash-slash comments from processing by stringify().
    ** 041 SV  20250408          Add help parameter usable with -?. ref 6859
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.preproc;

    import com.goldencode.ast.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.util.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.p2j.io.*;
    import com.goldencode.p2j.uast.*;

    import javax.xml.parsers.ParserConfigurationException;
    import java.io.*;
    import java.util.*;
    import java.util.logging.*;

    import antlr.collections.AST;
    import antlr.collections.impl.*;
    import antlr.debug.misc.*;
    import antlr.*;

    /**
     * Implements the external entry points into the Progress 4GL compatible
     * preprocessor and a collection of static utility methods.
     */
    public class Preprocessor
    implements ProgressParserTokenTypes
    {
       /** Logger */
       private static final ConversionStatus LOG = ConversionStatus.get(Preprocessor.class);

       /** shared environment */
       private Environment env = null;

       /** Syntax help text. */
       private final static String[] helptxt;

       static
       {
          ArrayList<String> array = new ArrayList<>();

          String[] ht = new String[]
          {
             "-----------------------Automated TTY Test Preprocess-----------------------",
             "",
             "java com.goldencode.p2j.preproc.Preprocessor [options] <file_to_preprocess>",
             "",
             "Options:",
             "",
             "-?                     = display this help screen",
             "-basepath              = the top-level directory under where all source code",
             "                         for the project can be found, defaults to '.'",
             "-batch                 = return 'yes' from the BATCH-MODE build-in, defauls to 'no'",
             "-casesensitive         = determines whether include file names are treated as case-sensitive",
             "                         defaults to 'true', override with 'false'",
             "-debug                 = turns on debug output",
             "-fileseparator <sep>   = specifies the separator string to use between each path component",
             "                         (directories and file names) in a fully-qualified or partially-qualified",
             "                         filename or path. Defaults to File.separator (the platform-specific file",
             "                         separator on the system on which conversion is running) if not specified.",
             "                         On UNIX or Linux this is a / (slash) and on Windows this is a \\ (backslash).",
             "-hints <filename>      = specifies the hints file",
             "-keeptildes            = preserves tildes on output",
             "-lexonly               = runs the lexer only",
             "-marker <nnn>          = specifies marker character; nnn would usually be a decimal value from 1-7",
             "                         or 16-31 or 127. These are all ASCII control characters and are unlikely",
             "                         to be encountered on the input stream. If this character is encountered",
             "                         on input, an exception will be thrown.",
             "-opsys <name>          = name specifies the value of the OPSYS built-in, defaults to 'UNIX'",
             "-out <filename>        = specifies the output file; STDOUT stream is used by default. See Note.",
             "-err <filename>        = specifies the error file; STDERR stream is used by default. See Note.",
             "-warn <filename        = specifies the warning file; STDERR stream is used by default. See Note.",
             "-pathseparator <sep>   = specifies the separator string to use between each component in a search",
             "                         path. Defaults to File.pathSeparator (the platform-specific path separator",
             "                         on the system on which conversion is running) if not specified. On UNIX",
             "                         or Linux this is a : (colon) and on Windows this is a ; (semicolon).",
             "-propath path:path...  = specifies paths to use as the PROPATH",
             "-pushback <nnn>        = specifies the pushback buffer size, defaults to 8192",
             "-unixescapes           = honors backslash as if it was a tilde",
             "-winsys <name          = treats backslash as a normal character (instead of as an escape character)",
             "-arch {32 | 64}        = name specifies the value of the PROCESS-ARCHITECTURE built-in",
             "                         defaults to 32",
             "",
             "Parameters:",
             "",
             "file_to_preprocess     = relative or absolute filename of the .([pPwW]|cls) 4GL code",
             "",
             "Note: Please note that the STDERR/STDOUT stream is a PrintStream which may inappropriately modify",
             "national language character on output, even if the default language environment is set properly",
             "If you are not using UTF-8 as the default environment, then it is likely that you MUST use",
             "err, out or warn parameters to avoid this problem.",
             ""
          };

          for (String next : ht)
          {
             array.add(next);
          }

          helptxt = array.toArray(new String[0]);
       }

       /**
        * Removes newlines from the character array.
        * <p>
        * The input array may have regular newlines (NLs) and/or escapes like
        * <code>CR CR</code> or <code>CR NL</code>.
        * <p>
        * Regular newlines are deleted. Escapes are processed. The first CR
        * character is deleted and the second character is kept in the array.
        *
        * @param    buf
        *           array to process
        * @param    size
        *           current size of array
        * @param    keepTildes
        *           <code>true</code> to bypass deletion of regular newlines.
        *
        * @return   The new size of the array.
        */
       public static int stripNL(char[] buf, int size, boolean keepTildes)
       {
          int i;
          int j;

          for (i = 0; i < size; i++)
          {
             // handle CR escapes: CRCR and CRNL
             if ((buf[i] == '\r') && (buf[i + 1] == '\r' || buf[i + 1] == '\n'))
             {
                // shift the remainder of the array one character to the left
                for (j = i + 1; j < size; j++)
                   buf[j - 1] = buf[j];

                // the remaining size of the array is smaller now
                size--;

                // as the buffer was moved to the left and this char consumed, we need to remain on the
                // same place in the array - otherwise a possible next NL will be missed
                i = i - 1;

                continue;
             }

             // don't modify non-newlines or even regular newlines if keep tildes
             // is specified
             if (buf[i] != '\n' || keepTildes)
                continue;

             // handle regular NLs

             // shift the remainder of the array one character to the left
             for (j = i + 1; j < size; j++)
                buf[j - 1] = buf[j];

             // as the buffer was moved to the left and this char consumed, we need to remain on the
             // same place in the array - otherwise a possible next NL will be missed
             i = i - 1;

             // the remaining size of the array is smaller now
             size--;
          }

          return size;
       }

       /**
        * Gets the specified positional parameter for the current input file.
        * Position 0 means the input file name as originally specified. The
        * result corresponds to the Progress' <code>{n}</code> reference.
        * <p>
        * This call marks the referenced positional argument as "used".
        *
        * @param    env
        *           instance of the preprocessor environment
        * @param    pos
        *           positional reference number as a <code>String</code>
        *
        * @return   substitution text for the referenced variable
        */
       public static String getPosArg(Environment env, String pos)
       throws NumberFormatException,
              IOException
       {
          int n = Integer.parseInt(pos);
          FileScope f = (FileScope)env.getSym().getScope();

          if (n < 0)
             throw new NumberFormatException();

          if (n == 0)
             return f.getFileName();

          String name = new String("{" + n);
          return getVariable(env, name);
       }

       /**
        * Gets a string of all positional parameters for the current input file.
        * The result corresponds to the Progress' <code>{*}</code> reference.
        * <p>
        * This call marks all positional arguments as "used".
        *
        * @param    env
        *           instance of the preprocessor environment
        *
        * @return   substitution text for the referenced variable
        */
       public static String getAllPos(Environment env)
       {
          FileScope f = (FileScope)env.getSym().getScope();

          // mark all positional arguments as used
          f.setAllPosUsed();

          // TODO: this does not generate a reference hint
          return f.getAllPositionalArguments();
       }

       /**
        * Gets a string of all named parameters for the current input file.
        * The result corresponds to the Progress' <code>{&amp;*}</code> reference.
        * <p>
        * This call marks all named arguments as "used".
        *
        * @param    env
        *           instance of the preprocessor environment
        *
        * @return   substitution text for the referenced variable
        */
       public static String getAllNamed(Environment env)
       {
          FileScope f = (FileScope)env.getSym().getScope();

          // mark all named arguments as used
          f.setAllNamedUsed();

          // TODO: this does not generate a reference hint
          return f.getAllNamedArguments();
       }

       /**
        * Gets the specified variable. The result corresponds to a Progress
        * <code>{&amp;name}</code> preprocessor reference. Note that for arguments,
        * only the currently defined value should be used. If the current scope
        * does not have a definition for an argument, it will NOT be found from
        * any deeper scope.
        * <p>
        * Any valid argument usage will mark the referenced argument as "used".
        * Likewise, name usage metrics will be cached in a hints file should one
        * be provided.
        *
        * @param    env
        *           instance of the preprocessor environment
        * @param    name
        *           name of the variable
        *
        * @return   substitution text for the referenced variable
        *
        * @throws   IOException
        *           when the pushback buffer is full
        */
       public static String getVariable(Environment env, String name)
       throws IOException
       {
          ScopedSymbolDictionary dict = env.getSym();

          Symbol    sym = (Symbol) dict.lookupSymbol(name);

          int    origin     = -1;
          String value      = null;
          String originText = null;

          if (sym == null)
          {
             origin     = Symbol.UNDEFINED;
             originText = "undefined";
             value      = "";
          }
          else
          {
             origin     = sym.getOrigin();
             originText = sym.getOriginText();

             if (origin == Symbol.BUILTIN)
             {
                BuiltinSymbol bsym = (BuiltinSymbol) sym;
                value = bsym.getValue(env);
             }
             else
             {
                value = sym.getValue();
             }
          }

          // safety code
          if (value == null)
             value = "";

          // ensure that arguments can only be accessed from the scope in which
          // they were defined
          if (origin == Symbol.ARGUMENT)
          {
             int depth = dict.locateSymbol(name);

             if (depth != 0)
             {
                value = "";
             }
             else
             {
                FileScope fs  = (FileScope) dict.getScope();
                fs.setArgUsed(name);
             }
          }

          // create reference hint
          Hints hintsObj = env.getHints();

          // hints are not meaningful in a preprocessor statement since that
          // does not generate code as a result (e.g. a global define does not
          // generate code but it can have arg references in its value portion)
          if (hintsObj != null && !env.isInStatement())
          {
             List hints = hintsObj.getHintsList();
             hints.add(new ReferenceHint(name, originText, value));
             env.queueMarker("r", hints.size() - 1);
          }

          return value;
       }

       /**
        * Includes a file into the input stream. A new <code>FileScope</code>
        * is created and added to a new scope in the dictionary. The input
        * stream is switched to the new file.
        *
        * @param    env
        *           instance of the preprocessor environment
        * @param    filename
        *           name of the file as specified in the inlcude file reference
        * @param    args
        *           Progress arguments to be given to the include file
        * @param    deferred
        *           Text that must be pushed back onto the stream AFTER the
        *           included file's stream reaches EOF.
        */
       public static void includeFile(Environment env,
                                      String      filename,
                                      Map         args,
                                      String      deferred)
       {
          // below is a special case of using inclusion mechanism for parsing
          // a list of Progress source file arguments
          if (filename.equals("."))
          {
             env.setMap(args);
             return;
          }

          FileScope   fs      = new FileScope(filename, args, env.getOpt(), true);
          CharScanner lex     = env.getLex();
          Options     options = env.getOpt();

          String resolvedFile = fs.resolveFileName(options.getProPath(), env);
          if (resolvedFile != null)
          {
             UastHints hints = new UastHints(resolvedFile);
             fs.setCharset(hints.getSourceCharset());
          }

          // try and open the include file stream
          if (resolvedFile == null || !fs.open(options.getProPath(), env))
          {
             env.wprint("Include file \"" + filename + "\" not found.",
                        lex.getLine(), lex.getColumn());

             // last ditch effort to find any unique match (even though the
             // match is not found in the propath); this may find a file that
             // is not correct, but it is probably no worse than bailing out and
             // in many cases (where the propath is wrong), it should be better
             if (!fs.attemptFallback(env))
             {
                return;
             }
             else
             {
                env.wprint("Include file \"" + fs.getFallbackName() + "\" found " +
                           "via fallback search!",
                           lex.getLine(), lex.getColumn());
             }
          }

          ClearStream ins = (ClearStream) env.getIns();
          MultiReader mlt = ins.getStream();

          if (ins.queueSize() > 0)
          {
             // there is already queued data waiting, we must aggregate it with
             // our other deferred data and clear the current queue, saving the
             // aggregate to be pushed back later

             // temporarily push back our additional buffered data
             env.pushBack(deferred);

             // now we read all the queued data and clear the queue
             deferred = new String(ins.clear());
          }

          // switch streams
          mlt.switchTo(fs.getStream());

          // create new scope in the symbol dictionary
          env.getSym().addScope(fs);

          // we are not a reference, we are an include file
          env.setFile(true);

          // put the arguments to this file if any
          fs.copyNamedArguments(env.getSym());

          // save the deferred text for pushback later
          fs.setDeferredPushback(deferred);

          // set up reporting
          fs.setSavedLine(lex.getLine());
          fs.setSavedColumn(lex.getColumn());

          // support -debug option
          if (env.getOpt().isDebug())
          {
             env.getSym().dumpCurrentScope("entering", env.getErr());
             ((ClearStream)env.getIns()).dumpState();
          }
       }

       /**
        * Clean up the incoming regressions from characters that interfere the conversion process with
        * stringify method.  The following fixups will be done:
        * <p>
        * Removing the new line characters.
        * <p>
        * The slash-slash comments will be removed.
        *
        * @param    expr
        *           The expression to fix.
        *
        * @return   The fixed expression without new line chars and slash-slash comments.
        */
       public static String cleanupExpr(String expr)
       {
          // allocate enough room to work without extra allocations
          StringBuilder sb = new StringBuilder(expr.length());

          // process line by line
          String [] exprArr = expr.split("\\R");
          for (int i = 0; i < exprArr.length; i++)
          {
             String nextStr = exprArr[i];
             if (nextStr != null)
             {
                int cNdx = nextStr.indexOf("//");
                if (cNdx == -1)
                {
                   // not a comment line add complete content
                   sb.append(nextStr);
                }
                else // cNdx >= 0
                {
                   // drop any slash-slash comments
                   if (cNdx > 0)
                   {
                      // use content before slash-slash comment starts
                      sb.append(nextStr.substring(0, cNdx));
                   }
                }
             }
          }

          return sb.toString();
       }

       /**
        * Use recursive lexing to identify any instances of the {@code defined()} preproc built-in
        * function and convert any text in between the LPARENS and the next RPARENS into a string
        * literal.  Yes, the 4GL just simply searches for the next RPARENS instead of treating the
        * contents as a possible sub-expression. This means that any kind of junk can be placed
        * there (invalid expressions, random text with spaces and punctuation).  We must not try
        * to parse it.  We also must not just lookup the first symbol that is seen (even though
        * some 4GL programmers probably think that is what happens, since customer code has been
        * seen that puts a sub-expression like {@code defined(some-incl-arg = "some text")}. The
        * 4GL just reports this as undefined and moves on. This method is meant to resolve the
        * inherent conflict in how we lex.  The contents of the parenthesis must not be lexed
        * normally.
        * <p>
        * This uses a real lexer to process the expression.  A simple approach of just searching on
        * the text "defined" won't work because it is valid to include that text inside comments and
        * string literals.
        *
        * @param    exp
        *           The expression to fix.
        * @param    env
        *           The current environment.
        * @param    cr
        *           The callback resolver to use. The state WILL NOT be modified on exit.
        *
        * @return   The fixed expression or the input expression if no defined builtin was found
        *           or if any error occurs.
        */
       public static String fixupDefined(String exp, Environment env, CallbackResolver cr)
       {
          // quick out if there is nothing to do
          if (exp.toLowerCase().indexOf("defined") == -1)
          {
             return exp;
          }

          // Zap any newlines and slash-slash comments, so they don't influence our stringify
          exp = cleanupExpr(exp);

          // initial setup (warning, schema dictionary support is disabled
          // but basic lvalue processing in the parser should still be safe
          // since the symbol resolver handles this case)
          SymbolResolver  sym    = new SymbolResolver(false, true, null, true, false, null);
          StringReader    expr   = new StringReader(exp);
          ProgressLexer   lexer  = new ProgressLexer(expr, sym);

          // make a copy so we don't mess up the state
          CallbackResolver cr2 = new CallbackResolver(cr);

          // populate the CR with the standard built-in functions
          // (this MUST be done after the lexer is initialized above otherwise
          // any conflicts will be blown away)
          ExpressionEvaluator.addStandardFunctions(sym, cr2);

          // populate SymbolResolver with all Keywords in the CallbackResolver
          cr2.populateSymbolResolver(sym);

          try
          {
             Token  tok   = null;
             String ltxt  = null;
             int    ltype = -1;
             int    ntype = -1;

             // read the token stream to find the start of the next defined() call
             do
             {
                ltype = ntype;
                ltxt  = (tok != null) ? tok.getText() : "";
                tok   = lexer.nextToken();
                ntype = tok.getType();

                if (ltype == FUNC_INT && ltxt.equalsIgnoreCase("defined") && ntype == LPARENS)
                {
                   int begin = tok.getColumn();         // 1-based index
                   int end   = exp.indexOf(')', begin); // 0-based index
                   int len   = exp.length();

                   if (end == -1)
                   {
                      // no closing parens, let it fail downstream
                      break;
                   }

                   // fixup the instance that was found and recurse if there is more to check
                   String front  = exp.substring(0, begin);
                   String middle = stringify(exp.substring(begin, end)) + ")";
                   String back   = (len == end + 1) ? ""
                                                    : fixupDefined(exp.substring(end + 1), env, cr);

                   return front + middle + back;
                }
             }
             while (tok != null && ntype != EOF);
          }

          catch (TokenStreamException tse)
          {
            // some unexpected failure, let it fail downstream
          }

          return exp;
       }

       /**
        * Evaluates Progress preprocessor expression given as a string.
        *
        * @param    env
        *           Shared environment.
        * @param    xpr
        *           Progress preprocessor expression, everything between
        *           <code>&amp;IF</code> or <code>&amp;ELSEIF</code> and
        *           <code>&amp;THEN</code>. If this is <code>null</code>, the
        *           empty string or only whitespace, then the result will be
        *           <code>false</code>.
        *
        * @return   <code>true</code> or <code>false</code> according to
        *           the Progress rules.
        */
       public static boolean evaluate(Environment env, String xpr)
       {
          // save any current version of the propath
          String old = EnvironmentOps.getSearchPathOverride();

          xpr = StringHelper.safeTrim(xpr);

          if (xpr == null || xpr.length() == 0)
          {
             return false;
          }

          try
          {
             CallbackResolver cr = new CallbackResolver();

             // create a callback for DEFINED() function
             cr.addCallback(new Keyword("defined", 0, FUNC_INT, false),
                            new DefinedHelper(env),
                            "com.goldencode.p2j.preproc.DefinedHelper",
                            "defined");

             // create a callback for PROCESS-ARCHITECTURE() function
             cr.addCallback(new Keyword("process-architecture", 0, VAR_INT, false),
                            env.getOpt(),
                            "com.goldencode.p2j.preproc.Options",
                            "getProcessArchitectureBDT");

             // all other functions are not preprocessor specific, so they are implemented inside
             // the ExpressionEvaluator

             // create callbacks for all defined preprocessor variables since
             // these can be referenced inside an expression as a normal variable
             // reference

             ScopedSymbolDictionary sym = env.getSym();

             Set     vars    = sym.symbolSet();
             String  var[]   = new String[vars.size()];
             Keyword varKW[] = new Keyword[vars.size()];

             // copy the data into the array
             vars.toArray(var);

             String type = null;

             for (int i = 0; i < var.length; i++)
             {
                varKW[i] = new Keyword(var[i], 0, VAR_INT, false);
                switch (((Symbol) sym.lookupSymbol(var[i])).getOrigin())
                {
                   case Symbol.GLOBAL:
                      type = "globalVariable";
                      break;
                   case Symbol.SCOPED:
                      type = "scopedVariable";
                      break;
                   case Symbol.ARGUMENT:

                      // only insert a callback for this argument if it is part
                      // of the current scope, otherwise ignore it
                      if (sym.locateSymbol(var[i]) != 0)
                         continue;

                      type = "includeFileArgument";
                      break;
                   case Symbol.BUILTIN:
                      type = "builtinVariable";
                      break;
                }

                // skip this one because otherwise we get an exception at eval time (builtinVariable instances
                // can't be accessed in the 4GL), but we have a separate helper already registered for this one
                // which is accessible in the 4GL
                if (var[i].equalsIgnoreCase("process-architecture"))
                {
                   continue;
                }

                cr.addCallback(varKW[i],
                               null,
                               "com.goldencode.p2j.preproc.Symbol",
                               type);
             }

             xpr = fixupDefined(xpr, env, cr);

             // reconstitute the propath from the split version
             StringBuilder sb = new StringBuilder();
             String[] list = env.getOpt().getProPath();

             for (int j = 0; j < list.length; j++)
             {
                if (j > 0)
                   sb.append(File.pathSeparator);

                sb.append(list[j]);
             }

             // setup the propath into the current context
             EnvironmentOps.setSearchPath(sb.toString(), true);

             // evaluate the expression
             BaseDataType result = ExpressionEvaluator.evaluateExpr(xpr, cr, false);

             if (result == null)
             {
                env.wprint("Preprocessor evaluation of expression '" + xpr + "' failed, returning false.",
                           env.getLex().getLine(),
                           env.getLex().getColumn());
                return false;
             }

             // interpret the results

             if (result instanceof logical)
             {
                return ((logical) result).booleanValue();
             }

             if (result instanceof NumberType)
             {
                return ((NumberType) result).intValue() > 0;
             }

             if (result instanceof character)
             {
                return TextOps.lengthOf((character) result) > 0;
             }

             env.wprint("Preprocessor expression returned illegal data type '" +
                        result.getClass().getName() + "', returning false.",
                        env.getLex().getLine(),
                        env.getLex().getColumn());
             return false;
          }

          catch (Exception excpt)
          {
             env.eprint("Exception caught in evaluation of expression '" +
                        xpr + "'!",
                        env.getLex().getLine(),
                        env.getLex().getColumn());
             LOG.log(Level.SEVERE, "", excpt);
          }

          finally
          {
             // restore the original override
             EnvironmentOps.setSearchPath(old, true);
          }

          return false;
       }

       /**
        * Preprocess the specified source file (this will not return until
        * preprocessing is complete).
        *
        * @param    topFile
        *           Progress source file to preprocess along with arguments.
        * @param    output
        *           Stream to use for preprocessed output.
        * @param    errors
        *           Stream to use for preprocessor error messages.
        * @param    warnings
        *           Stream to use for preprocessor warning messages.
        * @param    options
        *           Configuration values.
        *
        * @throws   PreprocessorException
        *           When syntactically incorrect input is encountered.
        */
       public Preprocessor(FileScope topFile,
                           Writer    output,
                           Writer    errors,
                           Writer    warnings,
                           Options   options)
       throws PreprocessorException
       {
          // initialize input and output
          MultiReader mlt = new MultiReader();
          ClearStream ins = new ClearStream(mlt, options);

          // create the global scope for the top level file
          ScopedSymbolDictionary sym = new ScopedSymbolDictionary(topFile);

          // initialize the pool of builtin variables
          BuiltinVariable.init(sym);

          // put the named arguments to this file if any
          topFile.copyNamedArguments(sym);

          // create the shared environment object
          // add the local scope
          sym.addScope(topFile);

          try
          {
             env = new Environment(null, ins, output, errors, warnings, sym, options);
          }

          catch (ParserConfigurationException pcx)
          {
             throw new PreprocessorException(pcx);
          }

          catch (IOException pio)
          {
             throw new PreprocessorException(pio);
          }

          ins.setEnvironment(env);

          // focus on the top level file
          if (topFile.open(options.getProPath(), env))
          {
             mlt.switchTo(topFile.getStream());
          }
          else
          {
             throw new PreprocessorException("Failed to open the file " +
                                             topFile.getFileName());
          }

          // attach text lexer to the input stream, which also creates
          // a shared input state object
          TextLexer lexer = new TextLexer(env);
          env.setLex(lexer);
          lexer.setFilename(topFile.getFile().getName());

          // create parser attached to selector
          TextParser parser = new TextParser(lexer, env);
          env.setPar(parser);
          parser.setFilename(topFile.getFile().getName());

          try
          {
             // kick off parsing
             if (options.isLexonly())
             {
                PrintWriter pw = null;

                if (output instanceof PrintWriter)
                {
                   pw = (PrintWriter) output;
                }
                else
                {
                   pw = new PrintWriter(output);
                }

                LexerDumpHelper test = new LexerDumpHelper(lexer,
                                                           TextParser._tokenNames,
                                                           pw);
                test.dump();
             }
             else
             {
                parser.text();

                env.addConstantDefinitions();

                if (options.isDebug())
                   sym.dump(env.getErr());

                int errCount = env.getErrorCounter();
                if (errCount > 0)
                {
                   throw new PreprocessorException("Preprocessor done with " +
                                                   errCount + " error(s).");
                }
             }
          }

          catch(PreprocessorException e)
          {
             throw e;
          }

          catch(Exception e)
          {
             env.eprint("Exception: "+e);
             ((ClearStream)env.getIns()).dumpState();
             env.eprint("Scanning: " + lexer.getText());

             try
             {
                env.eprint("Scanning: LA(1)='" + lexer.LA(1) + "' LA(2)='" +
                           lexer.LA(2) + "'");
             }

             catch (Exception ee)
             {
             }

             LOG.log(Level.SEVERE, "", e);
             sym.dump(env.getErr());
          }

          finally
          {
             topFile.close();

             try
             {
                Hints hintsObj = env.getHints();

                if (hintsObj != null)
                {
                   env.getHints().save();
                }
             }

             catch (IOException exc)
             {
                env.eprint("Exception while saving hints file: " + exc);
             }

             try
             {
                // it is not clear why we end up losing output in many cases, but
                // this will ensure that all pending output is not lost
                output.flush();
                errors.flush();

                if (warnings != null)
                {
                   warnings.flush();
                }
             }

             catch (IOException ioe)
             {
                // ignore
             }
          }
       }

       /**
        * Returns the Environment.
        *
        * @return   The Environment object associated with this Preprocessor.
        */
       public Environment getEnv()
       {
          return env;
       }

       /**
        * Command line preprocessor driver. Parses the command line, creates
        * an instance of the Options class, configures it with the parsed
        * values and calls the preprocessor.
        *
        * @param   args
        *          command line parameters. Options, if any, have to come
        *          first. Following this is the input file name and, possibly,
        *          Progress arguments.
        *          <p>
        *          The following options are recognized:
        *          <ul>
        *          <li><code>-basepath</code> the top-level directory under where
        *              all source code for the project can be found, defaults to
        *              ".";
        *          <li><code>-batch</code> return "yes" from the BATCH-MODE
        *              built-in, defaults to "no";
        *          <li><code>-casesensitive</code> determines whether include
        *              file names are treated as case-sensitive;  defaults to
        *              "true", override with "false".
        *          <li><code>-debug</code> turns on debug output;
        *          <li><code>-fileseparator sep</code> specifies the separator
        *              string to use between each path component (directories and
        *              file names) in a fully-qualified or partially-qualified
        *              filename or path. Defaults to <code>File.separator</code>
        *              (the platform-specific file separator on the system on which
        *              conversion is running) if not specified. On UNIX or Linux
        *              this is a / (slash) and on Windows this is a \ (backslash).
        *          <li><code>-hints filename</code> specifies the hints file;
        *          <li><code>-keeptildes</code> preserves tildes on output;
        *          <li><code>-lexonly</code> runs the lexer only;
        *          <li><code>-marker nnn</code> specifies marker character; nnn
        *              would usually be a decimal value from 1-7 or 16-31 or
        *              127.  These are all ASCII control characters and are
        *              unlikely to be encountered on the input stream. If this
        *              character is encountered on input, an exception will be
        *              thrown.
        *          <li><code>-opsys name</code> name specifies the value of the
        *              OPSYS built-in, defaults to "UNIX";
        *          <li><code>-out filename</code> specifies the output file;
        *              STDOUT stream is used by default; <b>Please note that
        *              the STDOUT stream is a <code>PrintStream</code> which
        *              may inappropriately modify national language characters
        *              on output, even if the default language environment is
        *              set properly.  If you are not using UTF-8 as the default
        *              environment, then it is likely that you MUST use this
        *              parameter to avoid this problem.</b>
        *          <li><code>-err filename</code> specifies the error file;
        *              STDERR stream is used by default; <b>Please note that
        *              the STDERR stream is a <code>PrintStream</code> which
        *              may inappropriately modify national language characters
        *              on output, even if the default language environment is
        *              set properly.  If you are not using UTF-8 as the default
        *              environment, then it is likely that you MUST use this
        *              parameter to avoid this problem.</b>
        *          <li><code>-warn filename</code> specifies the warning file;
        *              STDERR stream is used by default; <b>Please note that
        *              the STDERR stream is a <code>PrintStream</code> which
        *              may inappropriately modify national language characters
        *              on output, even if the default language environment is
        *              set properly.  If you are not using UTF-8 as the default
        *              environment, then it is likely that you MUST use this
        *              parameter to avoid this problem.</b>
        *          <li><code>-pathseparator sep</code> specifies the separator
        *              string to use between each component in a search path.
        *              Defaults to <code>File.pathSeparator</code> (the
        *              platform-specific path separator on the system on which
        *              conversion is running) if not specified. On UNIX or Linux
        *              this is a : (colon) and on Windows this is a ; (semicolon).
        *          <li><code>-propath path:path...</code> specifies paths to use
        *              as the PROPATH;
        *          <li><code>-pushback nnn</code> specifies the pushback buffer
        *              size; defaults to 8192.
        *          <li><code>-unixescapes</code> honors backslash as if it was
        *              a tilde;
        *          <li><code>-windowsescapes</code> treats backslash as a normal
        *              character (instead of as an escape character);
        *          <li><code>-winsys name</code> name specifies the value of the
        *              WINDOW-SYSTEM built-in, defaults to "TTY";
        *          <li><code>-arch {32 | 64 }</code> name specifies the value of the
        *              PROCESS-ARCHITECTURE built-in, defaults to 32;
        *          </ul>
        */
       public static void main(String[] args)
       {
          int       rc   = 0;
          FileScope fs   = null;
          Writer    os   = new PrintWriter(System.out);
          Writer    err  = new PrintWriter(System.err);
          Writer    warn = err;

          // create and configure default options
          Options opts = new Options();

          // first, identify and process options
          int i = 0;

          optsloop:
          for (i = 0; i < args.length; i++)
          {
             if ("-?".equals(args[i]))
             {
                if (helptxt != null)
                {
                   for (int j = 0; j < helptxt.length; j++)
                   {
                      System.err.println(helptxt[j]);
                   }
                }
                System.exit(0);
             }
             if ("-keeptildes".equals(args[i]))
             {
                opts.setKeepTildes(true);
             }
             else if ("-unixescapes".equals(args[i]))
             {
                opts.setUnixEscapes(true);
             }
             else if ("-windowsescapes".equals(args[i]))
             {
                opts.setUnixEscapes(false);
             }
             else if ("-batch".equals(args[i]))
             {
                opts.setBatchMode(true);
             }
             else if ("-casesensitive".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   String val = args[i + 1];
                   boolean caseSens = "true".equals(val);
                   if (!caseSens && !"false".equals(val))
                   {
                      LOG.log(Level.SEVERE,
                              "Unrecognized '-casesenstive' value:  '" + val + "';  must be true or false");
                      System.exit(1);
                   }

                   opts.setCaseSensitive(caseSens);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line.  Include file  case sensitivity " +
                      "parameter (true/false) expected.");
                   System.exit(1);
                }
             }
             else if ("-fileseparator".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setFileSeparator(args[i + 1]);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. File separator character expected.");
                   System.exit(1);
                }
             }
             else if ("-pathseparator".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setPathSeparator(args[i + 1]);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Path separator character expected.");
                   System.exit(1);
                }
             }
             else if ("-opsys".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setOpsys(args[i + 1]);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Operating System name expected.");
                   System.exit(1);
                }
             }
             else if ("-winsys".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setWindowSystem(args[i + 1]);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Window System name expected.");
                   System.exit(1);
                }
             }
             else if ("-arch".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setProcessArchitecture(Integer.parseInt(args[i + 1]));
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Process Architecture bit size expected.");
                   System.exit(1);
                }
             }
             else if ("-out".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   try
                   {
                      os = new BufferedWriter(new FileWriter(args[i + 1]));
                   }
                   catch (Exception e)
                   {
                      // ignore failure and write to STDOUT
                   }
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Output file name expected.");
                   System.exit(1);
                }
             }
             else if ("-err".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   try
                   {
                      err = new BufferedWriter(new FileWriter(args[i + 1]));
                   }
                   catch (Exception e)
                   {
                      // ignore failure and write to STDERR
                   }
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Error file name expected.");
                   System.exit(1);
                }
             }
             else if ("-warn".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   try
                   {
                      warn = new BufferedWriter(new FileWriter(args[i + 1]));
                   }
                   catch (Exception e)
                   {
                      // ignore failure and write to STDERR
                   }
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Warning file name expected.");
                   System.exit(1);
                }
             }
             else if ("-propath".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   String[] paths = args[i + 1].split(File.pathSeparator);
                   i++;
                   opts.setProPath(paths);
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. List of paths expected.");
                   System.exit(1);
                }
             }
             else if ("-basepath".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setBasepath(args[i + 1]);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line.Basepath expected.");
                   System.exit(1);
                }
             }
             else if ("-pushback".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   int n = Integer.parseInt(args[i + 1]);
                   i++;

                   if (n <= 0)
                   {
                      LOG.log(Level.SEVERE, "Pushback value must be a positive + number.");
                      System.exit(1);
                   }
                   opts.setPushbackDepth(n);
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Pushback value expected.");
                   System.exit(1);
                }
             }
             else if ("-debug".equals(args[i]))
             {
                opts.setDebug(true);
             }
             else if ("-lexonly".equals(args[i]))
             {
                opts.setLexonly(true);
             }
             else if ("-hints".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   opts.setHintsFile(args[i + 1]);
                   i++;
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Hints file name expected.");
                   System.exit(1);
                }
             }
             else if ("-marker".equals(args[i]))
             {
                if (i < args.length - 1)
                {
                   int n = Integer.parseInt(args[i + 1]);
                   i++;
                   if ((n >= 8 && n <= 13) || (n >= 32 && n <= 127))
                   {
                      LOG.log(Level.SEVERE, "Marker code invalid.");
                      System.exit(1);
                   }
                   opts.setMarker((char)n);
                }
                else
                {
                   LOG.log(Level.SEVERE, "Premature end of command line. Marker character code expected.");
                   System.exit(1);
                }
             }
             else
             {
                break;
             }
          }

          // now process the input file and, possibly, arguments
          if (i == args.length)
          {
             LOG.log(Level.SEVERE, "Premature end of command line. Input file expected.");
             System.exit(1);
          }

          StringBuilder cmdLine = new StringBuilder("{ ");
          for (; i < args.length; i++)
          {
             cmdLine.append(args[i] + " ");
          }
          cmdLine.append("}");

          String cmd = cmdLine.toString();

          if (opts.isDebug())
          {
             LOG.log(Level.SEVERE, "Virtual file: " + cmdLine);
          }

          // create an input stream for the virtual file
          StringReader virt = new StringReader(cmd);

          try
          {
             // IMPORTANT! The name of the virtual file in the FileScope
             // constructor call below should remain "/". This signifies a name
             // that NEVER can be a filename. ClearStream checks the name to see
             // if the top level scope is for this virtual name and then bypasses
             // appending an extra space character, which is otherwise required.
             fs = new FileScope("/", virt);
             Preprocessor pp = new Preprocessor(fs, os, err, warn, opts);
          }

          catch (PreprocessorException e)
          {
             LOG.log(Level.INFO, "Preprocessor done with errors.", e);
             rc = -1;
          }

          catch (Exception e)
          {
             LOG.log(Level.SEVERE, "Exception: ", e);
             rc = -2;
          }

          if (rc == 0)
             LOG.log(Level.INFO, "Preprocessor done.");

          System.exit(rc);
       }

       /**
        * Rewrite the input string as a string literal, ensuring that any unescaped double quote
        * characters are properly escaped.  This is not for general purpose usage, but it is
        * safe enough for the internal usage in fixing the defined() builtin.
        *
        * @param    input
        *           String to process, possibly empty string but must not be {@code null}.
        *
        * @return   The quote encapsulated string literal.  The result will properly lex as a STRING.
        */
       private static String stringify(String input)
       {
          if (input.length() == 0)
          {
             return input;
          }

          // zap ~ and \ (the possible escape chars) while doubling any double quote chars
          input = input.replace("\"", "\"\"").replace("~", "").replace("\\", "");

          return "\"" + StringHelper.safeTrim(input) + "\"";
       }
    }