# LexerDumpFilter.java

    /*
    ** Module   : LexerDumpFilter.java
    ** Abstract : wraps a lexer token stream, acting as a filter this writes the tokens to file
    **
    ** Copyright (c) 2004-2022, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ---------------------------Description----------------------------
    ** 001 GES 20050221   @19919 First version, with support for:
    **                           - driving the lexer and for each token:
    **                             - decoding token types
    **                             - writing output with line numbers, token types and text
    **                           - transparently operates as if the ProgressLexer was being used
    **                             directly
    ** 002 GES 20050725   @21826 Added code to hide tokens that used to be dropped by the lexer,
    **                           but which are now hidden downstream (between the lexer and the
    **                           parser) after us.
    ** 003 GES 20080324   @37663 Removed invalid constructors. Converted to readers from streams to
    **                           support I18N.
    ** 004 GES 20090424   @41956 Import change.
    ** 005 GES 20100119   @44543 Made sure to close output when done otherwise the output can be
    **                           truncated.
    ** 006 GES 20150102          Added flushing after each token is written. This solves a problem
    **                           with empty output files that at least occurred when there was an
    **                           exception.  We also see this problem at times when there is no
    **                           exception and this may also solve that issue.
    ** 007 TJD 20220504          Java 11 compatibility minor changes
    */
    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.uast;

    import com.goldencode.util.*;
    import java.io.*;
    import java.text.*;
    import antlr.*;

    /**
     * Provides a simple mechanism to run the ProgressLexer on a given Progress
     * 4GL source file and to save the resulting stream of tokens while still
     * servicing the needs of the ProgressParser.  To accomplish this, a filter
     * approach is used to intercept the calls to {@link #nextToken} and write
     * the token to file before then returning the token as normal.  Since this
     * class is a subclass of ProgressLexer, it allows a transparent override
     * of the <code>nextToken</code> without breaking any of the other lexer
     * interfaces being used by the caller.
     * <p>
     * The output is a column formatted list with one line written for each
     * token in the resulting stream.  The output is written to a file of the
     * caller's choice.
     * <p>
     * For a source file that includes the following input on line 8:
     * <pre>
     * define variable hello as character initial ?.
     * </pre>
     * The output format (for line 8) looks as follows:
     * <pre>
     * [00008:001] &lt;KW_DEFINE&gt;             define
     * [00008:008] &lt;KW_VAR&gt;                variable
     * [00008:017] &lt;SYMBOL&gt;                hello
     * [00008:023] &lt;KW_AS&gt;                 as
     * [00008:026] &lt;KW_CHAR&gt;               character
     * [00008:036] &lt;KW_INIT&gt;               initial
     * [00008:044] &lt;UNKNOWN_VAL&gt;           ?
     * [00008:045] &lt;DOT&gt;                   .
     * </pre>
     * The first column is interpreted as "[" line number ":" column number "]".
     * The second column is the symbolic representation of the integer token
     * type of that token.  The third column is the actual text from the file
     * that corresponds to that token.
     */
    public class LexerDumpFilter
    extends ProgressLexer
    {
       /** Size in characters of the largest possible token type name. */
       private int maxSymSize = 0;

       /** Formats a line number padded on left with zeros. */
       private NumberFormat lfmt = NumberFormat.getInstance();

       /** Formats a column number padded on left with zeros. */
       private NumberFormat cfmt = NumberFormat.getInstance();

       /** Output to which the tokens will be written. */
       private PrintWriter out = null;

       /** Flag to indicate if automatic closure of the stream is required. */
       private boolean cleanup = false;

       /** Last token type processed. */
       private int lastType = -1;

       /** Flag to indicate if skip processing should be bypassed. Useful to see hidden tokens. */
       private boolean bypassSkip = false;

       /**
        * Constructs a lexer using a reader as input and associating a
        * <code>SymbolResolver</code> with the instance, will open an output
        * stream using the passed <code>File</code> object.  User must call
        * {@link #setOutput} before starting to lex, otherwise no output
        * will be created.
        *
        * @param    in
        *           The reader representing the Progress 4GL source code.
        * @param    sr
        *           Provides keyword dictionary storage and lookup for symbol
        *           resolution in the lexer.
        */
       public LexerDumpFilter(Reader in, SymbolResolver sr)
       throws IllegalArgumentException,
              IOException
       {
          super(in, sr);
          initialize();
       }

       /**
        * Constructs a lexer using a reader as input and associating a
        * <code>SymbolResolver</code> with the instance, will open an output
        * stream using the passed <code>File</code> object.
        *
        * @param    in
        *           The reader representing the Progress 4GL source code.
        * @param    sr
        *           Provides keyword dictionary storage and lookup for symbol
        *           resolution in the lexer.
        * @param    file
        *           The output file to which the tokens will be written.
        */
       public LexerDumpFilter(Reader in, SymbolResolver sr, File file)
       throws IllegalArgumentException,
              IOException
       {
          this(in, sr);
          openOutput(file);
       }

       /**
        * This is the main filter point which calls the parent lexer, obtains
        * the next token, writes it to the output file and then returns it to
        * the caller.
        *
        * @return    The next token from the parent lexer.
        */
       public Token nextToken()
       throws TokenStreamException
       {
          CommonToken token = null;

          try
          {
             token = (CommonToken) super.nextToken();

             // here is our filter!
             writeToken(token);
          }

          finally
          {
             out.flush();
          }

          return token;
       }

       /**
        * Allow this instance to cleanup any resources that were allocated.
        */
       public void close()
       {
          synchronized (this)
          {
             if (cleanup)
             {
                out.close();
                cleanup = false;
             }
          }
       }

       /**
        * Set the flag to determine if normally skipped tokens are written to output or not.
        *
        * @param    bypass
        *           <code>true</code> to write normally skipped tokens to the output file.
        */
       public void setBypassSkip(boolean bypass)
       {
          bypassSkip = bypass;
       }

       /**
        * A looping driver to simulate a real client, which outputs a report with
        * one line per token in the token stream.
        *
        * @throws   antlr.TokenStreamException
        */
       public void dump() throws antlr.TokenStreamException
       {
          CommonToken next;

          do
          {
             next = (CommonToken) nextToken();
          }
          while (next.getType() != Token.EOF_TYPE);
       }

       /**
        * Allows the current output to be switched for a user-defined
        * stream.  The caller must close this stream when done!
        *
        * @param    out
        *           The output destination.
        */
       public synchronized void setOutput(PrintWriter out)
       {
          if (out != null)
          {
             this.out = out;
          }
       }

       /**
        * Cleanup resources for this instance, in this case the output stream
        * is closed if it was instantiated by this class.
        */
       @SuppressWarnings("deprecation")
       protected void finalize()
       throws Throwable
       {
          synchronized (this)
          {
             if (cleanup)
             {
                out.close();
                cleanup = false;
             }
          }

          super.finalize();
       }

       /**
        * Setup and calculate formatting defaults.
        *
        * @param    file
        *           The output file to which the tokens will be written.
        */
       private synchronized void openOutput(File file)
       throws IllegalArgumentException,
              IOException
       {
          if (file != null)
          {
             // if this filename exists as a directory, we can't save results
             if (file.isDirectory())
             {
                throw new IllegalArgumentException(file.toString() +
                                                   " cannot be a directory.");
             }
             else
             {
                // remove the target file if it exists.
                if (file.exists())
                {
                   file.delete();
                }

                out     = new PrintWriter(new FileWriter(file));
                cleanup = true;
             }
          }
          else
          {
             throw new IllegalArgumentException("Output file cannot be null.");
          }
       }

       /**
        * Setup and calculate formatting defaults.
        */
       private void initialize()
       {
          lfmt.setMinimumIntegerDigits(5);
          lfmt.setGroupingUsed(false);
          cfmt.setMinimumIntegerDigits(3);
          cfmt.setGroupingUsed(false);

          calcMaximumSymbolNameSize();
       }

       /**
        * Scans the list of token type names (text symbols that represent each
        * integer token type) and calculates the width in characters of the
        * largest one.  This is saved as the column width in the output report.
        */
       private void calcMaximumSymbolNameSize()
       {
          for (int i = 0; i < ProgressParser._tokenNames.length; i++)
          {
             int current = ProgressParser._tokenNames[i].length();
             if (current > maxSymSize)
                maxSymSize = current;
          }

          // make room for the angle brackets and an extra space for padding
          maxSymSize += 3;
       }

       /**
        * Writes a one line record for the token object passed as a parameter.
        * The record is printed to the output stream in a 3 column format as
        * documented in the overview section.
        * <p>
        * WARNING: this method does nothing when the output stream is
        * <code>null</code>.
        *
        * @param    next
        *           The token to decode and write to file.
        */
       private synchronized void writeToken(CommonToken next)
       {
          if (out != null)
          {
             int type = next.getType();

             // consume a second EOF
             if (type == ProgressParser.EOF && lastType == ProgressParser.EOF)
             {
                return;
             }

             // hack to ignore fields that will be returned by the lexer in
             // hidden mode (when being used with the ProgressParser) BUT
             // which are supposed to be skipped
             if ((type != ProgressParser.WS         &&
                  type != ProgressParser.COMMENT    &&
                  type != ProgressParser.TILDE      &&
                  type != ProgressParser.BACKSLASH) ||  bypassSkip)
             {
                String decode = ProgressParser.lookupTokenName(type);

                decode = StringHelper.leftAlignText(decode + ">", maxSymSize);

                StringBuilder sb = new StringBuilder();
                sb.append("[")
                  .append(lfmt.format(next.getLine()))
                  .append(":")
                  .append(cfmt.format(next.getColumn()))
                  .append("] <")
                  .append(decode)
                  .append(" ")
                  .append(next.getText());

                out.println(sb.toString());
             }

             // save off the token type for next time
             lastType = type;
          }
       }
    }