LexerDumpFilter.java
/*
** Module : LexerDumpFilter.java
** Abstract : wraps a lexer token stream, acting as a filter this writes the tokens to file
**
** Copyright (c) 2004-2022, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ---------------------------Description----------------------------
** 001 GES 20050221 @19919 First version, with support for:
** - driving the lexer and for each token:
** - decoding token types
** - writing output with line numbers, token types and text
** - transparently operates as if the ProgressLexer was being used
** directly
** 002 GES 20050725 @21826 Added code to hide tokens that used to be dropped by the lexer,
** but which are now hidden downstream (between the lexer and the
** parser) after us.
** 003 GES 20080324 @37663 Removed invalid constructors. Converted to readers from streams to
** support I18N.
** 004 GES 20090424 @41956 Import change.
** 005 GES 20100119 @44543 Made sure to close output when done otherwise the output can be
** truncated.
** 006 GES 20150102 Added flushing after each token is written. This solves a problem
** with empty output files that at least occurred when there was an
** exception. We also see this problem at times when there is no
** exception and this may also solve that issue.
** 007 TJD 20220504 Java 11 compatibility minor changes
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.uast;
import com.goldencode.util.*;
import java.io.*;
import java.text.*;
import antlr.*;
/**
* Provides a simple mechanism to run the ProgressLexer on a given Progress
* 4GL source file and to save the resulting stream of tokens while still
* servicing the needs of the ProgressParser. To accomplish this, a filter
* approach is used to intercept the calls to {@link #nextToken} and write
* the token to file before then returning the token as normal. Since this
* class is a subclass of ProgressLexer, it allows a transparent override
* of the <code>nextToken</code> without breaking any of the other lexer
* interfaces being used by the caller.
* <p>
* The output is a column formatted list with one line written for each
* token in the resulting stream. The output is written to a file of the
* caller's choice.
* <p>
* For a source file that includes the following input on line 8:
* <pre>
* define variable hello as character initial ?.
* </pre>
* The output format (for line 8) looks as follows:
* <pre>
* [00008:001] <KW_DEFINE> define
* [00008:008] <KW_VAR> variable
* [00008:017] <SYMBOL> hello
* [00008:023] <KW_AS> as
* [00008:026] <KW_CHAR> character
* [00008:036] <KW_INIT> initial
* [00008:044] <UNKNOWN_VAL> ?
* [00008:045] <DOT> .
* </pre>
* The first column is interpreted as "[" line number ":" column number "]".
* The second column is the symbolic representation of the integer token
* type of that token. The third column is the actual text from the file
* that corresponds to that token.
*/
public class LexerDumpFilter
extends ProgressLexer
{
/** Size in characters of the largest possible token type name. */
private int maxSymSize = 0;
/** Formats a line number padded on left with zeros. */
private NumberFormat lfmt = NumberFormat.getInstance();
/** Formats a column number padded on left with zeros. */
private NumberFormat cfmt = NumberFormat.getInstance();
/** Output to which the tokens will be written. */
private PrintWriter out = null;
/** Flag to indicate if automatic closure of the stream is required. */
private boolean cleanup = false;
/** Last token type processed. */
private int lastType = -1;
/** Flag to indicate if skip processing should be bypassed. Useful to see hidden tokens. */
private boolean bypassSkip = false;
/**
* Constructs a lexer using a reader as input and associating a
* <code>SymbolResolver</code> with the instance, will open an output
* stream using the passed <code>File</code> object. User must call
* {@link #setOutput} before starting to lex, otherwise no output
* will be created.
*
* @param in
* The reader representing the Progress 4GL source code.
* @param sr
* Provides keyword dictionary storage and lookup for symbol
* resolution in the lexer.
*/
public LexerDumpFilter(Reader in, SymbolResolver sr)
throws IllegalArgumentException,
IOException
{
super(in, sr);
initialize();
}
/**
* Constructs a lexer using a reader as input and associating a
* <code>SymbolResolver</code> with the instance, will open an output
* stream using the passed <code>File</code> object.
*
* @param in
* The reader representing the Progress 4GL source code.
* @param sr
* Provides keyword dictionary storage and lookup for symbol
* resolution in the lexer.
* @param file
* The output file to which the tokens will be written.
*/
public LexerDumpFilter(Reader in, SymbolResolver sr, File file)
throws IllegalArgumentException,
IOException
{
this(in, sr);
openOutput(file);
}
/**
* This is the main filter point which calls the parent lexer, obtains
* the next token, writes it to the output file and then returns it to
* the caller.
*
* @return The next token from the parent lexer.
*/
public Token nextToken()
throws TokenStreamException
{
CommonToken token = null;
try
{
token = (CommonToken) super.nextToken();
// here is our filter!
writeToken(token);
}
finally
{
out.flush();
}
return token;
}
/**
* Allow this instance to cleanup any resources that were allocated.
*/
public void close()
{
synchronized (this)
{
if (cleanup)
{
out.close();
cleanup = false;
}
}
}
/**
* Set the flag to determine if normally skipped tokens are written to output or not.
*
* @param bypass
* <code>true</code> to write normally skipped tokens to the output file.
*/
public void setBypassSkip(boolean bypass)
{
bypassSkip = bypass;
}
/**
* A looping driver to simulate a real client, which outputs a report with
* one line per token in the token stream.
*
* @throws antlr.TokenStreamException
*/
public void dump() throws antlr.TokenStreamException
{
CommonToken next;
do
{
next = (CommonToken) nextToken();
}
while (next.getType() != Token.EOF_TYPE);
}
/**
* Allows the current output to be switched for a user-defined
* stream. The caller must close this stream when done!
*
* @param out
* The output destination.
*/
public synchronized void setOutput(PrintWriter out)
{
if (out != null)
{
this.out = out;
}
}
/**
* Cleanup resources for this instance, in this case the output stream
* is closed if it was instantiated by this class.
*/
@SuppressWarnings("deprecation")
protected void finalize()
throws Throwable
{
synchronized (this)
{
if (cleanup)
{
out.close();
cleanup = false;
}
}
super.finalize();
}
/**
* Setup and calculate formatting defaults.
*
* @param file
* The output file to which the tokens will be written.
*/
private synchronized void openOutput(File file)
throws IllegalArgumentException,
IOException
{
if (file != null)
{
// if this filename exists as a directory, we can't save results
if (file.isDirectory())
{
throw new IllegalArgumentException(file.toString() +
" cannot be a directory.");
}
else
{
// remove the target file if it exists.
if (file.exists())
{
file.delete();
}
out = new PrintWriter(new FileWriter(file));
cleanup = true;
}
}
else
{
throw new IllegalArgumentException("Output file cannot be null.");
}
}
/**
* Setup and calculate formatting defaults.
*/
private void initialize()
{
lfmt.setMinimumIntegerDigits(5);
lfmt.setGroupingUsed(false);
cfmt.setMinimumIntegerDigits(3);
cfmt.setGroupingUsed(false);
calcMaximumSymbolNameSize();
}
/**
* Scans the list of token type names (text symbols that represent each
* integer token type) and calculates the width in characters of the
* largest one. This is saved as the column width in the output report.
*/
private void calcMaximumSymbolNameSize()
{
for (int i = 0; i < ProgressParser._tokenNames.length; i++)
{
int current = ProgressParser._tokenNames[i].length();
if (current > maxSymSize)
maxSymSize = current;
}
// make room for the angle brackets and an extra space for padding
maxSymSize += 3;
}
/**
* Writes a one line record for the token object passed as a parameter.
* The record is printed to the output stream in a 3 column format as
* documented in the overview section.
* <p>
* WARNING: this method does nothing when the output stream is
* <code>null</code>.
*
* @param next
* The token to decode and write to file.
*/
private synchronized void writeToken(CommonToken next)
{
if (out != null)
{
int type = next.getType();
// consume a second EOF
if (type == ProgressParser.EOF && lastType == ProgressParser.EOF)
{
return;
}
// hack to ignore fields that will be returned by the lexer in
// hidden mode (when being used with the ProgressParser) BUT
// which are supposed to be skipped
if ((type != ProgressParser.WS &&
type != ProgressParser.COMMENT &&
type != ProgressParser.TILDE &&
type != ProgressParser.BACKSLASH) || bypassSkip)
{
String decode = ProgressParser.lookupTokenName(type);
decode = StringHelper.leftAlignText(decode + ">", maxSymSize);
StringBuilder sb = new StringBuilder();
sb.append("[")
.append(lfmt.format(next.getLine()))
.append(":")
.append(cfmt.format(next.getColumn()))
.append("] <")
.append(decode)
.append(" ")
.append(next.getText());
out.println(sb.toString());
}
// save off the token type for next time
lastType = type;
}
}
}