Quoter.java
/*
** Module : Quoter.java
** Abstract : Java implementation of the Quoter utility
**
** Copyright (c) 2012-2023, Golden Code Development Corporation.
**
** -#- -I- --Date-- ----------------Description-----------------
** 001 CS 20121214 Created initial version.
** 002 EVL 20160224 Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
** 003 RFB 20230214 Changes related to ref. #7024:
** - The actual INPUT_NOT_AVAILABLE_ERR message from 4GL does not have a
** colon in it. The INCOMPATIBLE_OPTIONS_ERR string had an extra space.
** - The delimited tokens need to take into account they could contain delimiters.
** These will be surrounded by double-quotes (not single) regardless of what the
** 4GL documentation indicates. The quoter-utility.p testcases bear this out.
** - Must take into account quoted columns which may have multiple or unbalanced
** quotation marks in a given column, as well as continue past the end of line.
** RFB 20230301 Post-review updates:
** - Re-add protection against NPE in quoted()
** - Simplification of the quoteCountWorker
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.util;
import java.io.*;
import java.util.*;
/**
* A class implementation of the Progress QUOTER utility.
* <p>
* This class is used to parse the quoter utility parameters and execute the
* equivalent logic of the Progress utility.
* <p>
* The quoter utility will put the elements identified and extracted from
* input between double quotes (in the case where a double quote is found
* inside it will be doubled) the result being sent to output which may be a
* stream or a file.
* <p>
* The quoter utility can be called with the following parameters:
* <ul>
* <li> Output is always written to System.out.
* <li>An input file can be specified as argument with full or relative
* path.
* <p> If specified, the input will be taken from the file, and in case
* the file doesn't exist or the file does not have read permission,
* an error will be sent to output announcing the source cannot be read.
* <p> If the file parameter is not specified the input will be taken
* from System.in.
* <li> Delimiter option (optional): <b>-D</b> followed by the
* delimiter character.
* <p> When this option is used the input is read line by line and each
* line will be split using the specified delimiter before being sent
* to output.
* <p>Ex: <b>OS-COMMAND QUOTER /home/table.csv -D , >/home/table.q</b>
* <p>Delimiter and column options can never be used together!
* <li> Column range option (optional) : <b>-C</b> followed by the column
* ranges.
* <p> The column ranges must be separated in the form <b>start-end</b>
* and separated by: <b>,</b> . No spaces must be present!
* <p> This option will read input line by line and if presented with
* valid column ranges will split the line into columns from the
* starting position of the column range to the end position of the
* column range (Ex of column range: <b>1-4,5-15,...</b> the first
* column will start from the first character up to the 4'th character
* included).
* <p>Usage Ex: <b>
* OS-COMMAND QUOTER /home/table.csv -C 1-2,4-20,22-26 >/home/table.q
* </b>
* <p>Delimiter and column options can never be used together!
* <li> When there is no delimiter or column option the quoter will read
* the input line by line , quote it and output it.
* </ul>
* <p><b>Error Handling.</b>
* <p>When errors are encountered quoter will send the error message to the
* output, then quit.
* <p>The following errors messages will be output:
* <ul>
* <li> <b>Unknown option:</b> + option :
* <p>This error is sent to output when quoter is called with an
* option different from -C or -D.
* <li> <b>Quoter only accepts one input file</b>
* <p>This error is sent to output when there are more than one
* arguments that are not options or options attributes.
* <li> <b>The -c option should be followed by a space and then a list of
* column ranges, separated by commas</b>
* <p>This error is sent to output when -c option doesn't have a
* value after the option.
* <li> <b>The -d option should be followed by a space and then a single
* character</b>
* <p>This error is sent to output when -d option doesn't have a value
* after the option or that value is not a single character.
* <li> <b>Invalid range: "</b> + invalid range.
* <p>This error is sent when an invalid range is detected within the
* column ranges: This can happen because of the starting index being
* greater than the end index or, one or both of the indexes are not
* a number.
* <li> <b>Unable to read: </b> + input file.
* <p>This error is sent when the input file cannot be read (doesn't
* exist or the user doesn't have read permission).
* <li> <b>Incompatible options: -c and -d</b>
* <p>This error is sent when quoter is used with both column and
* delimiter options.
* </ul>
*/
public class Quoter
{
/** Error for invalid column range. */
private static final String INVALID_RANGE_ERR = "Invalid range: ";
/** Error message for unknown option. */
private static final String UNKNOWN_OPTION_ERR = "Unknown option: ";
/** Error message for multiple inputs. */
private static final String INPUT_ERR =
"Quoter only accepts one input file";
/** Error message for missing column range */
private static final String COLL_ERR =
"The -c option should be followed by a space and then a list of " +
"column ranges, separated by commas";
/** Error message for missing delimiter. */
private static final String DELIM_ERR =
"The -d option should be followed by a space and then a single " +
"character";
/** Error message for when input is not available for read. */
private static final String INPUT_NOT_AVAILABLE_ERR = "Unable to read ";
/**
* Error message for when both -C and -D options are used at the same time.
*/
private static final String INCOMPATIBLE_OPTIONS_ERR =
"Incompatible options: -c and -d";
/** quoter detection variable. */
private static final String QUOTER = "QUOTER";
/** quoter.exe detection variable*/
private static final String QUOTER_EXE = "QUOTER.EXE";
/** Columns option indicator. */
private static final String COLUMNS_OPTION = "-C";
/** Delimiter option indicator. */
private static final String DELIMITER_OPTION = "-D";
/** The quote character used to delimit output strings. */
private static final String QUOTE = "\"";
/**
* The escaped quote character (by doubling it) used inside output strings.
*/
private static final String QUOTE_2X = QUOTE + QUOTE;
/**
* Private constructor to stop class instantiation
*/
private Quoter()
{
}
/**
* The entry point method of the Quoter class.
* <p>This will check for valid parameters and option usage of quoter
* before calling the mainm quoter logic.
*
* @param args
* The parameters given to the quoter utility
*/
public static void main(String args[])
{
String inFile = null;
String delimiter = null;
String colRanges = null;
String errorMessage = null;
// interpret args (no args is still a valid case)
if (args != null && args.length > 0)
{
for (int i = 0; i < args.length; i++)
{
if (isOption(args[i]))
{
if (args[i].toUpperCase().equals(DELIMITER_OPTION))
{
// check to see if there is a value for the option
if (i + 1 < args.length)
{
delimiter = args[i + 1];
// skip this arg for checking
i++;
}
else
{
errorMessage = DELIM_ERR;
// get out of the "for" processing loop when the first
// error is encountered
break;
}
}
else if (args[i].toUpperCase().equals(COLUMNS_OPTION))
{
// check to see if there is a value for the option
if (i + 1 < args.length)
{
colRanges = args[i + 1];
// skip this arg for checking
i++;
}
// no column option value
else
{
errorMessage = COLL_ERR;
// get out of the "for" processing loop when the first
// error is encountered
break;
}
}
else
{
errorMessage = UNKNOWN_OPTION_ERR + args[i];
// get out of the "for" processing loop when the first
// error is encountered
break;
}
}
else
{
if (inFile == null)
{
inFile = args[i];
}
else
{
errorMessage = INPUT_ERR;
// get out of the "for" processing loop when
// error is encountered
break;
}
}
}
}
// check for delimiter error and output it if another error hasn't been
// detected before
if (delimiter != null && delimiter.length() > 1 && errorMessage == null)
{
errorMessage = DELIM_ERR;
}
BufferedReader inputReader = null;
BufferedWriter outputWriter =
new BufferedWriter(new OutputStreamWriter(System.out));
// check to see if any errors have been discovered
if (errorMessage != null)
{
try
{
outputWriter.write(errorMessage);
outputWriter.close();
}
catch (IOException e)
{
// nothing to be done but return
}
return;
}
if (inFile == null)
{
inputReader = new BufferedReader(new InputStreamReader(System.in));
}
else
{
try
{
inputReader = new BufferedReader(new FileReader(inFile));
}
catch (FileNotFoundException e)
{
try
{
outputWriter.write(INPUT_NOT_AVAILABLE_ERR + inFile);
outputWriter.close();
}
catch (IOException ioe)
{
return;
}
return;
}
}
try
{
process(inputReader, outputWriter, delimiter, colRanges);
}
catch (Exception e)
{
// nothing to do
}
finally
{
if (inputReader != null)
{
tryClose(inputReader);
}
tryClose(outputWriter);
}
}
/**
* Utility method that checks if a string argument is a call to the quoter
* utility ot not.
*
* @param command
* The <code>String</code> argument which is to be tested for
* identifying it as a quoter argument or not.
* @return Returns <code>true</code> if the parameter is identified as
* a call to quoter utility or <code>false</code> otherwise.
*/
public static boolean isQuoterCommand(String command)
{
// case insensitive matching
command = command.toUpperCase();
// the only possible matches must always end with one of these two
// values
if (command.endsWith(QUOTER) || command.endsWith(QUOTER_EXE))
{
int start = command.lastIndexOf(QUOTER);
if (start == 0)
{
// no pathing or trash in front is a match
return true;
}
else
{
// something is in front, check if it is a path
char pathSep = command.charAt(start - 1);
if (pathSep == '/' || pathSep == '\\')
{
return true;
}
// at this point there must be trash in front, this isn't valid
}
}
// all the other case will return false
return false;
}
/**
* Utility method that checks for option's values and process them before
* calling the main quoter logic.
*
* @param inputReader
* The input file name
* @param outputWriter
* The output file name
* @param delimiter
* The delimiter if any
* @param colRange
* The column ranges to be parsed
*
* @throws Exception
* In case something goes wrong.This signals abort.
*/
private static void process(BufferedReader inputReader,
BufferedWriter outputWriter,
String delimiter,
String colRange)
throws Exception
{
String separator = null;
List<ColumnRange> columnRanges = null;
if (delimiter != null && colRange != null)
{
// We cannot have both column and delimiter options
outputWriter.write(INCOMPATIBLE_OPTIONS_ERR);
throw new Exception();
}
if (delimiter != null)
{
// Progress behavior if separator is not a character it will
// take only the first character of the String.
separator = delimiter.substring(0, 1);
delModeWorker(inputReader, outputWriter, separator);
}
else
{
// get column ranges if any.
columnRanges = getColumnRange(colRange, outputWriter);
if ((columnRanges != null) && (columnRanges.size() > 0))
{
colModeWorker(inputReader, outputWriter, columnRanges);
}
else
{
simModeWorker(inputReader, outputWriter);
}
}
}
/**
* Utility method that writes into the destination file the line of quoted
* tokens (row of columns parsed from the input file).
*
* @param tokens
* The <code>array</code> of <code>String</code> values
* representing a row of columns parsed from the input file
* @param writer
* The <code>BufferedWriter</code> used to output the quoted
* characters.
*
* @throws IOException
* In case something went wrong in the writing process.
* <p>This indicates that the main quoter method should close
* opened resources and terminate.
*/
private static void writeDelimitedLine(String[] tokens,
BufferedWriter writer)
throws IOException
{
for (int i = 0; i < tokens.length; i++)
{
if (i != 0)
{
writer.write(" ");
}
// empty tokens are replaced with hyphens ('-') in the output, unless it ends the line
writer.write(tokens[i].length() > 0 ? quote(tokens[i])
: (i+1) < tokens.length ? "-"
: "");
}
}
/**
* Utility method that quotes a <code>String</code> using double-quote and
* doubles any double quote it finds inside inside the <code>String</code>
* variable
*
* @param quoted
* The <code>String</code> to be quoted.
*
* @return The quoted <code>String</code> value.
*/
private static String quote(String quoted)
{
if (quoted != null)
{
// Handle an edge case where a single quote is left to append
String qString = (quoted.length() == 1 && quoted.charAt(0) == '\"') ? ""
: quoted.replaceAll(QUOTE, QUOTE_2X);
return '"' + qString + '"';
}
else
{
return null;
}
}
/**
* Utility method to see if a string starts with - and can be a command
* line option.
*
* @param string
* The <code>String</code> to be checked
*
* @return <code>true</code> if string starts with - , <code>false</code>
* otherwise
*/
private static boolean isOption(String string)
{
return string.startsWith("-");
}
/**
* Utility method that identifies the columns from an input
* <code>String</code> line extracts and quotes them then outputs them to
* the destination file
*
* @param line
* The <code>String</code> line representing a row from the input
* file.
* @param columnRanges
* The <code>List</code> of <code>ColumnRange</code> used to
* identify the column from the input line.
* @param writer
* The <code>BufferedWriter</code> used to output the quoted
* characters.
*
* @throws IOException
* In case something went wrong in the writing process.
* <p>This indicates that the main quoter method should close
* opened resources and terminate.
*/
private static void writeColumnsLine(String line,
List<ColumnRange> columnRanges,
BufferedWriter writer)
throws IOException
{
boolean first = true;
for (ColumnRange column : columnRanges)
{
String columnText = null;
if (line.length() >= column.getEndColumn())
{
columnText = line.substring(column.getStartColumn(),
column.getEndColumn());
}
else
{
if (column.getStartColumn() > line.length())
{
columnText = null;
}
else
{
columnText = line.substring(column.getStartColumn());
}
}
if (first)
{
first = false;
}
else
{
writer.write(" ");
}
if (columnText != null)
{
writer.write(quote(columnText));
}
else
{
// when column text is null Progres puts a "-" mark
writer.write("-");
}
}
}
/**
* Utility method that parses a <code>String</code> containing column range
* and returns the <code>List</code> of <code>ColumnRange</code> that
* contains the character range from a line where the column can be found.
*
* @param colRange
* The <code>String</code> representation of the column ranges.
* @param destFile
* The <code>BufferedWriter</code> used to output the quoted
* characters or any error messages in this case
*
* @return The parsed <code>List</code> of <code>ColumnRange</code>.
*
* @throws Exception
* In case there are invalid column ranges.
* <p>This indicates that the main quoter method should close
* opened resources and terminate.
*/
private static List<ColumnRange> getColumnRange(String colRange,
BufferedWriter destFile)
throws Exception
{
List<ColumnRange> columnRanges = null;
if (colRange != null)
{
columnRanges = new ArrayList<ColumnRange>();
String[] ranges = colRange.split(",");
for (int i = 0; i < ranges.length; i++)
{
String[] columns = ranges[i].split("-");
if (columns.length == 2)
{
int startCol = -1;
int endCol = -1;
try
{
startCol = Integer.parseInt(columns[0]);
endCol = Integer.parseInt(columns[1]);
}
catch (NumberFormatException e)
{
// column range not a number
// output error to destination file
destFile.write(INVALID_RANGE_ERR + ranges[i]);
// Invalid range - Throw exception to mark end
throw new Exception();
}
// check for invalid column ranges
if (startCol > endCol)
{
if (destFile != null)
{
// output error to destination file
destFile.write(INVALID_RANGE_ERR + ranges[i]);
// Invalid range - Throw exception to mark end
throw new Exception();
}
}
else
{
columnRanges.add(new ColumnRange(startCol, endCol));
}
}
}
}
return columnRanges;
}
/**
* Utility method that determines number of consecutive quotation marks
*
* @param line
* The <code>String</code> input line being checked.
* @param index
* The <code>int</code> location to start checking from.
*
* @return <code>integer</code> count of consecutive quotations marks.
*/
private static int quoteCountWorker(String line, int index)
{
int quoteCount = 0;
while (index < line.length() && line.charAt(index) == '"')
{
quoteCount++;
index++;
}
return quoteCount;
}
/**
* Utility method that breaks up the tokens in a line by the delimiter
*
* @param readFile
* The input <code>BufferedReader</code> from which the info
* will be read.
* @param line
* The <code>String</code> input line to be broken up.
* @param delimiter
* The <code>String</code> character that is used to split the
* information from a line into columns.
*
* @return <code>String</code> array of the delimited columns.
*/
private static String[] delimitWorker(BufferedReader readFile,
String line,
String delimiter)
{
// Double quotes are used when the delimiter character is within the quoted string
List<String> result = new LinkedList<>();
boolean inColumn = false;
StringBuilder sb = new StringBuilder();
char delimiterCharacter = delimiter.charAt(0);
int end = line.length();
boolean more_quotes = false; // Indicates we have multiple quotes within a column
int quotes; // We use the quoteCountWorker to count the number of consecutive quotes
try
{
do // Outer loop handles columns split across lines
{
for (int i = 0; i < end; i++)
{
char current = line.charAt(i);
char next = (i < (end-1)) ? line.charAt(i+1) : '\0';
boolean newColumn = (sb.length() == 0);
boolean next_is_quote = (next == '\"');
if (current == '\"')
{
// Quotations marks need to be handled to determine if
// we are in a column containing delimiter characters
quotes = quoteCountWorker(line, i);
if (newColumn)
{
if (quotes % 2 == 1)
{
// An odd number of quotes starts inQuote mode
inColumn = !inColumn;
}
else
{
// We have extra quotations marks to deal with in pairs
more_quotes = true;
sb.append(current);
i++;
}
}
else
{
// Handle remaining portion of a column being processed
if (inColumn)
{
// Determine if the next character is a quote or not
if (next_is_quote)
{
sb.append(current);
i++;
}
else
{
// Next character is not a quote, so it we exit the quote section,
// but include conditionally if it is not the delimiter
inColumn = !inColumn;
if (next != delimiterCharacter)
{
sb.append(current);
}
}
}
else
{
// We aren't in a quote, include the current quote, but check for extras
sb.append(current);
if (more_quotes)
{
i++;
more_quotes = false;
}
}
}
}
else if (!inColumn && (current == delimiterCharacter))
{
// Snap off the column as we are not longer in a quoted portion of the line
result.add(sb.toString());
sb = new StringBuilder();
inColumn = false;
more_quotes = false;
}
else
{
// not an escaped char or special case
sb.append(current);
}
}
if (inColumn)
{
sb.append(System.getProperty("line.separator"));
if ((line = readFile.readLine()) == null)
{
inColumn = false;
}
else
{
end = line.length();
}
}
} while (inColumn);
result.add(sb.toString());
}
catch (IOException e)
{
// Best effort
}
return result.toArray(new String[0]);
}
/**
* Utility method that handles the logic of quoter utility for column mode.
*
* @param readFile
* The input <code>BufferedReader</code> from which the info
* will be read.
* @param destFile
* The destination <code>BufferedWriter</code> file to which the
* quoted lines will be output.
* @param columnRanges
* The <code>List</code> of <code>ColumnRange</code> used to
* identify the column from the input line.
*
* @return <code>true</code> if operation was successful or
* <code>false</code> if the operation failed at some point.
*/
private static boolean colModeWorker(BufferedReader readFile,
BufferedWriter destFile,
List<ColumnRange> columnRanges)
{
boolean first = true;
try
{
String line = null;
while ((line = readFile.readLine()) != null)
{
if (!first)
{
destFile.newLine();
}
else
{
first = false;
}
writeColumnsLine(line, columnRanges, destFile);
destFile.flush();
}
destFile.newLine();
destFile.flush();
}
catch (IOException e)
{
return false;
}
return true;
}
/**
* Utility method that handles the logic of quoter utility for delimiters.
*
* @param readFile
* The input <code>BufferedReader</code> from which the info
* will be read.
* @param destFile
* The destination <code>BufferedWriter</code> file to which the
* quoted lines will be output.
* @param delimiter
* The <code>String</code> character that is used to split the
* information from a line into columns.
*
* @return <code>true</code> if operation was successful or
* <code>false</code> if the operation failed at some point.
*/
private static boolean delModeWorker(BufferedReader readFile,
BufferedWriter destFile,
String delimiter)
{
boolean first = true;
try
{
String line = null;
while ((line = readFile.readLine()) != null)
{
if (first)
{
first = false;
}
else
{
destFile.newLine();
}
String[] tokens = delimitWorker(readFile, line, delimiter);
writeDelimitedLine(tokens, destFile);
destFile.flush();
}
destFile.newLine();
destFile.flush();
}
catch (IOException e)
{
return false;
}
return true;
}
/**
* Utility method that handles the logic of quoter utility for neither delimiter nor column mode.
*
* @param readFile
* The input <code>BufferedReader</code> from which the info
* will be read.
* @param destFile
* The destination <code>BufferedWriter</code> file to which the
* quoted lines will be output.
*
* @return <code>true</code> if operation was successful or
* <code>false</code> if the operation failed at some point.
*/
private static boolean simModeWorker(BufferedReader readFile,
BufferedWriter destFile)
{
boolean first = true;
try
{
String line = null;
while ((line = readFile.readLine()) != null)
{
if (!first)
{
destFile.newLine();
}
else
{
first = false;
}
destFile.write(quote(line));
destFile.flush();
}
destFile.newLine();
destFile.flush();
}
catch (IOException e)
{
return false;
}
return true;
}
/**
* Utility method that closes the given stream.
*
* @param stream
* The stream to be closed.
*/
private static void tryClose(Closeable stream)
{
try
{
stream.close();
}
catch (IOException ioe)
{
// ignore, this is a best efforts close
}
}
/**
* Utility class to handle column ranges
*/
private static class ColumnRange
{
/**
* This represents the starting point from where a column will be
* subtracted from a <code> String </code> line.
*/
private int startColumn;
/**
* This represents the end point to where a column will be subtracted
* from a <code> String </code> line.
*/
private int endColumn;
/**
* Constructor with 2 parameters. Which will set the start and end index
* defining a column.
*
* @param startColumn
* The index point from where the column starts(position
* starts from 1).
* @param endColumn
* The end index to where the column ends (inclusive).
*/
public ColumnRange(int startColumn, int endColumn)
{
// Java starts from 0 not from 1
this.startColumn = startColumn - 1;
// in Progress endColumn is inclusive so no need to subtract
this.endColumn = endColumn;
}
/**
* Returns the starting index of the column.
*
* @return The Java equivalent starting index of the column
*/
public int getStartColumn()
{
return startColumn;
}
/**
* Returns the end index of the column.
* @return The Java equivalent end index of the column.
*/
public int getEndColumn()
{
return endColumn;
}
}
}