ReportDriver.java
/*
** Module : ReportDriver.java
** Abstract : drives the pattern engine based on a list of report definitions
**
** Copyright (c) 2007-2023, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
** 001 GES 20070514 @33498 Created initial version that provides
** support for command line processing based
** on a list of report definitions.
** 002 GES 20070626 @34286 Added support for naming the master report
** index and source file index.
** 003 GES 20090424 @41934 Import change.
** 004 GES 20090515 @42212 Another import change.
** 005 GES 20130203 Rewrite to process all reports in a single pattern
** engine run. This cuts the total time of a reporting
** run down by 80%! As part of this, the input format
** (for defining the reports) has been streamlined and
** the reports are parsed into a list of report
** definition objects.
** 006 ECF 20150715 Replace StringBuffer with StringBuilder.
** 007 GES 20160311 Added supportLvlExpr parsing and assignment into the report def.
** 008 CA 20170118 Added -T option, which is like -S but with a trailing of explicit
** schema AST file names.
** 009 ECF 20170505 Retrofit for use with FWD Analytics web application.
** 010 GES 20210707 Added ignore list support.
** OM 20210907 Added new, much simplified syntax which allows the file list parameters to
** be automatically detected from the configuration file.
** OM 20210913 Create list of processed files based on flags read from AST registry.
** OM 20210923 Added -P<name> command line option for specifying the configuration profile.
** GES 20211221 Fixed abend when using new syntax with no non-option args.
** OM 20220405 XmlFilePlugin can load either a file or a stream from application's jar.
** GES 20220324 Match method name change.
** CA 20220501 Each profile has the same structure as the main 'global' config. Allow multiple
** profiles to be ran at once, with the conversion switching the state between
** profiles, when a resource (like a file or namespace) is being processed. Only
** front phase is supported at this time.
** 011 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.report;
import com.goldencode.ast.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.pattern.*;
import com.goldencode.p2j.uast.*;
import com.goldencode.p2j.util.*;
import com.goldencode.util.*;
import org.xml.sax.*;
import javax.xml.parsers.*;
import java.io.*;
import java.util.*;
import java.util.logging.*;
/**
* Special-purpose driver for the pattern engine to process a list of report definitions against a given set
* of ASTs. This provides a command line interface, please see {@link #main} for the syntax.
* <p>
* The report definitions are stored in an XML file with the following structure:
* <p>
* <ul>
* <li>list (required root element, contains a list of all reports to create)
* <ul>
* <li>idxname (optional attribute, contains the base file name of the master report index to be
* generated, defaults to "index" which would result in the master report filename being
* "index.html", when overridden this value will also be prepended to the source file
* cross-reference default filename "source_file_index.html")
* <li>report (element, 1 or more, represents a single report)
* <ul>
* <li>name (required attribute, contains the rule-set file name that creates the associated report)
* <li>descr (optional attribute, contains the description of the report)
* <li>variable (element, 0 or more, represents a user-replaceable input for the report being run)
* <ul>
* <li>name (required attribute, the name of the variable being set)
* <li>value (required attribute, the value to assign to the variable)
* </ul>
* </ul>
* </ul>
* </ul>
* <p>
* Report definition files have a {@code .rpt} filename suffix and must reside in the pattern path.
*/
public class ReportDriver
implements DebugLevels,
XmlConstants,
ReportConstants
{
/** Default admin password system property key */
private static final String APP_ADMIN_PASS = "appAdminPass";
/** Reporting rule-set. */
private static final String ruleset = "reports/consolidated_reports";
/** Database mode flag. */
private static boolean database = false;
/** Logger */
private static final ConversionStatus LOG = ConversionStatus.get(ReportDriver.class);
/**
* Run the pattern engine with the given input.
*
* @param files
* List of ASTs to process.
* @param dbg
* Debug level.
* @param init
* Variable initializers to enable.
*/
public static void patternEngine(FileList[] files, int dbg, Map<String, Object> init)
throws Exception
{
PatternEngine engine = new PatternEngine();
for (FileList fl : files)
{
if (fl != null)
{
engine.addAstSpec(fl);
}
}
PatternEngine.setDebugLevel(dbg);
engine.setReadOnly(true);
engine.setVariableInitializers(init);
engine.run(ruleset);
}
/**
* Read the given report definitions and store the list of reports for subsequent processing.
* Read both the standard reports as well as any user-defined reports that have been exported.
*
* @param rptdef
* File name root of the XML file storing the list of report definitions.
*
* @return The list of report definitions.
*
* @throws ConfigurationException
* if there is an error loading the project configuration.
* @throws ParserConfigurationException
* if there is an error configuring the XML parser.
* @throws SAXException
* if the XML SAX parser encounters an error.
* @throws IOException
* if there is an error accessing the file.
*/
private static List<ReportDefinition> loadReports(String rptdef)
throws ConfigurationException,
ParserConfigurationException,
IOException,
SAXException
{
ConfigurationPersistence cfgPers = new ConfigurationPersistence();
List<ReportDefinition> reports = new ArrayList<>();
// load standard reports
File file = ConfigLoader.findConfigurationFile(rptdef, RPT_SUFFIX);
int typePrimary = cfgPers.readXmlReports(file, reports);
String userRptBase;
switch (typePrimary)
{
case SRC_CODE_CACHE:
database = false;
userRptBase = "code";
break;
case SRC_SCHEMA:
database = true;
userRptBase = "schema";
break;
default:
throw new RuntimeException("Unknown report type: " + typePrimary);
}
// load user-defined reports, if any
file = new File(Configuration.home() +
File.separator +
String.format(AUTO_EXPORT_RPT_SPEC, userRptBase));
int typeUser = cfgPers.readXmlReports(file, reports);
// sanity: make sure user report type, if any, matches primary report type
if (typeUser != SRC_UNKNOWN && typeUser != typePrimary)
{
throw new RuntimeException("Unexpected report type found in " + file + ": " + typeUser);
}
return reports;
}
/**
* Provides a command line interface for generating a list of reports using a list of ASTs and a report
* definition.
* <p>
* Syntax:<br>
* <pre>
* {@code java -DP2J_HOME=<home> ReportDriver [options]}
* </pre>
* <p>
* Deprecated syntax:<br>
* <pre>
* {@code java -DP2J_HOME=<home> ReportDriver [options] <rptdef> <filelist>}
* {@code java -DP2J_HOME=<home> ReportDriver -S[options] <rptdef> <directory> <filespec>}
* {@code java -DP2J_HOME=<home> ReportDriver -T[options] <rptdef> <directory> <filespec> <filelist>}
* </pre>
* Where:
* <ul>
* <li>Options
* <ul>
* <li>{@code Dn} = set debug level 'n' (must be a numeric digit between
* {@code MSG_NONE} (0) and {@code MSG_TRACE} (3) inclusive
* <li>{@code S} = use {@code filespecs} instead of an explicit file list
* <li>{@code T} = use {@code filespecs} with a trailing explicit file list (optional)
* <li>{@code X} = use a {@code filespec} plus an {@code ignore_list}
* <li>{@code N} = no recursion in {@code filespec} or {@code ignore_list} mode
* </ul>
* <li>{@code rptdef} is the name of a report definition file found via the pattern path
* <li>explicit file list mode
* <ul>
* <li>{@code <filelist>} is arbitrary list of absolute and/or relative file names to scan
* </ul>
* <li>{@code <filespec>} mode
* <ul>
* <li>{@code <directory>} is the directory to search for files
* <li>{@code <filespec>} is the file specification that will be used to create a list of files to
* process, should be enclosed in double quotes if any of the wildcard characters '*' or '?'
* are used
* </ul>
* <li>{@code <filespec> + <ignore_list>} mode
* <ul>
* <li>{@code <directory>} is the directory to search for files
* <li>{@code <filespec>} is the file specification that will be used to create a list of files to
* process, should be enclosed in double quotes if any of the wildcard characters '*' or '?'
* are used
* <li>{@code <ignore_list>} is the file name which includes the list of directories/files to ignore
* </ul>
* </ul>
* <p>
* For the new syntax form (first) the {@code S}, {@code X} and {@code T} modes are unavailable. In this
* case the full set of parameters are extracted from the configuration file ({@code p2j.cfg.xml}). This is
* the recommended form to be used because it is much cleaner.<br>
* The other forms are prone to errors when passing in the parameters, but they are the solution if a
* partial report is needed.
*
* @param args
* List of command line arguments.
*/
public static void main(String[] args)
throws ConfigurationException
{
// option defaults
Options opt = processOptions(args);
// set the configuration profile
if (opt.cfgProfile != null)
{
Configuration.setDefaultProfiles(opt.cfgProfile);
}
if (opt.newSyntax)
{
newSyntax(opt);
}
else
{
oldSyntax(opt, args);
}
}
/**
* Emit a syntax statement and optional error message to
* <code>stderr</code>, then exit the process with the specified return
* code. This method is called from {@link #main} when the user passes
* command line parameters which are not viable.
*
* @param msg
* Optional error message; may be <code>null</code>.
* @param rc
* Process return code used with <code>System.exit()</code>.
*/
private static void syntax(String msg, int rc)
{
if (msg != null)
{
LOG.log(Level.SEVERE, msg);
}
String[] staticText =
{
"Syntax:",
" java -DP2J_HOME=<home> ReportDriver [options]",
"",
"Deprecated (but supported) syntax:",
" java -DP2J_HOME=<home> ReportDriver [options] <rptdef> <filelist>",
" java -DP2J_HOME=<home> ReportDriver -S[options] <rptdef> <directory> \"<filespec>\"",
" java -DP2J_HOME=<home> ReportDriver -T[options] <rptdef> <directory> \"<filespec>\" <filelist>",
"",
"Where",
" Options:",
" Dn = set DEBUG level 'n' (must be a numeric digit between " + MSG_NONE + " and " +
MSG_TRACE + " inclusive)",
" S = use a file SPECIFIER instead of an explicit file list",
" T = use filespecs and a TRAIL of explicit files",
" N = NO recursion in filespec mode or trail mode",
" Pname = define the configuration PROFILE",
"",
" rptdef = the report definition file to use, must be found somewhere in the pattern path",
" filelist = arbitrary list of absolute and/or relative file names to scan (default approach)",
" directory = a relative or absolute path (only used when using filespecs)",
" filespec = file filter specification of the filenames in the given directory to scan.",
" Should be enclosed in double quotes if any of the wildcard characters '*'",
" or '?' are used (only used when using filespecs)",
};
LOG.log(Level.WARNING, String.join(System.lineSeparator(), staticText));
System.exit(rc);
}
/**
* Read and validates the options sent in command line.
*
* @param args
* The command-line arguments.
*
* @return A structure which groups all needed parameters.
*/
private static Options processOptions(String[] args)
{
Options opt = new Options();
if (args.length == 0)
{
return opt;
}
// process options
if (!args[0].startsWith("-"))
{
opt.newSyntax = false;
return opt;
}
if (args.length >= 2)
{
opt.newSyntax = false;
}
opt.idx = 0;
while (opt.idx < args.length && args[opt.idx].startsWith("-"))
{
String opts = args[opt.idx].substring(1).toLowerCase(); // drop the '-' prefix
opt.idx++;
if (opts.startsWith("p"))
{
// this must be a single option token because the schema configuration profile name might contain
// characters which interfere with other options
opt.cfgProfile.add(opts.substring(1)); // drop the 'p'
continue;
}
if (opts.indexOf('s') != -1)
{
opt.explicit = false;
opt.filespec = true;
opt.newSyntax = false;
}
if (opts.indexOf('t') != -1)
{
opt.explicit = true;
opt.filespec = true;
opt.newSyntax = false;
}
if (opts.indexOf('x') != -1)
{
opt.explicit = false;
opt.filespec = false;
opt.ignore = true;
opt.newSyntax = false;
}
opt.recursion = opts.indexOf('n') == -1;
int dbg = opts.indexOf('d');
if (dbg != -1)
{
char level = opts.charAt(dbg + 1);
if (!Character.isDigit(level))
{
syntax("Missing numeric debug level character following D option.", -2);
}
else
{
opt.debug = Character.digit(level, 10);
if (opt.debug < MSG_NONE || opt.debug > MSG_TRACE)
{
syntax("Debug level must be between " + MSG_NONE + " and " + MSG_TRACE + ".", -3);
}
}
}
}
return opt;
}
/**
* Parse parameters according to old syntax and execute the report accordingly.
*
* @param opt
* The options as they were extracted from the first parameter.
* @param args
* The full set of parameters from command line.
*/
private static void oldSyntax(Options opt, String[] args)
{
// the deprecated mode requires (mode + directory + filespec) OR (mode and a filelist)
if (args.length < 2)
{
syntax(null, -1);
}
// process the report definition
if (opt.idx >= args.length)
{
syntax("Missing report definition parameter.", -4);
}
try
{
List<ReportDefinition> reports = loadReports(args[opt.idx++]);
FileList[] files = new FileList[2];
// at this point idx points to the next arg after the options, this must be one of our various forms
// of filename input
if (opt.filespec)
{
// there must be exactly 2 args left, unless explicit mode is used, too
int remArgs = args.length - opt.idx;
if ((!opt.explicit && remArgs != 2) || (opt.explicit && remArgs < 2))
{
syntax("Missing/incomplete filespec parameter.", -7);
}
files[0] = new FileSpecList(new File(args[opt.idx]), args[opt.idx + 1], opt.recursion);
opt.idx += 2;
}
if (opt.explicit)
{
// there must be at least 1 arg left
if (args.length <= opt.idx)
{
if (!opt.filespec)
{
// mandatory only if filespec was not included
syntax("Missing file list parameter.", -6);
}
}
else
{
int len = args.length;
String[] filelist = new String[len - opt.idx];
System.arraycopy(args, opt.idx, filelist, 0, len - opt.idx);
files[1] = new ExplicitFileList(filelist);
}
}
if (opt.ignore)
{
// there must be exactly 3 args left
int remArgs = args.length - opt.idx;
if (remArgs != 3)
{
syntax("Missing/incomplete filespec + ignore list parameter.", -8);
}
files[0] = FileListFactory.createBlacklist(args[opt.idx],
args[opt.idx + 1],
opt.recursion,
false,
args[opt.idx + 2]);
if (files[0] == null)
{
syntax("Failure creating ignore list.", -9);
}
else
{
// TODO: this won't work for .dict files
files[0] = new ExplicitFileList(files[0].withSuffix(".ast"));
}
}
Map<String, Object> vars = new HashMap<>();
String appAdminPass = System.getProperty(APP_ADMIN_PASS);
if (appAdminPass != null && appAdminPass.length() > 0)
{
vars.put(APP_ADMIN_PASS, '"' + StringHelper.stripEnclosing(appAdminPass, '"') + '"');
}
// add a definition for database mode
vars.put("databaseMode", database);
// add our reports
vars.put("reports", reports);
// create a new pattern engine, configure it and run
patternEngine(files, opt.debug, vars);
}
catch (Exception excpt)
{
LOG.log(Level.SEVERE, "", excpt);
}
}
/**
* Parse parameters according to new syntax and execute the two passes with appropriate parameters.
*
* @param opt
* The options as they were extracted from the optional first parameter.
*/
private static void newSyntax(Options opt)
{
// make sure the ast manager is set up and initialized
String name = Configuration.getParameter("registry");
AstManager.initialize(new XmlFilePlugin(Configuration.forceHome(name), false));
// 1. ReportDriver [-TDn] reports/schema *.dict
String[] dicts = AstManager.get().getFiles(AstGenerator.FLAG_DICT);
// we need to skip the "standard" schema which contains the _meta structure
int standardIdx = -1;
for (int i = 0; i < dicts.length; i++)
{
String dict = dicts[i];
if (dict.endsWith("/standard.dict") || dict.endsWith("\\standard.dict"))
{
standardIdx = i;
break;
}
}
if (standardIdx != -1)
{
String[] dicts2 = new String[dicts.length - 1];
System.arraycopy(dicts, 0, dicts2, 0, standardIdx);
System.arraycopy(dicts, standardIdx + 1, dicts2, standardIdx, dicts.length - standardIdx - 1);
dicts = dicts2;
}
doPass(opt.debug, "reports/schema", dicts);
// 2. ReportDriver [-SDn] reports/profile *.ast
String[] files = AstManager.get().getFiles(AstManager.FLAG_ORIGINAL);
for (int i = 0; i < files.length; i++)
{
files[i] += ".ast";
}
doPass(opt.debug, "reports/profile", files);
}
/**
* Does one of the passes.
* @param debugLevel
* The debug level to be used.
* @param rptDef
* The report definition.
* @param files
* The list of files to be processed.
*/
private static void doPass(int debugLevel, String rptDef, String[] files)
{
try
{
List<ReportDefinition> reports = loadReports(rptDef);
FileList[] fileLists = new FileList[1];
fileLists[0] = new ExplicitFileList(files);
Map<String, Object> vars = new HashMap<>();
String appAdminPass = System.getProperty(APP_ADMIN_PASS);
if (appAdminPass != null && appAdminPass.length() > 0)
{
vars.put(APP_ADMIN_PASS, '"' + StringHelper.stripEnclosing(appAdminPass, '"') + '"');
}
// add a definition for database mode
vars.put("databaseMode", database);
// add our reports
vars.put("reports", reports);
// create a new pattern engine, configure it and run
patternEngine(fileLists, debugLevel, vars);
}
catch (Exception excpt)
{
LOG.log(Level.SEVERE, "", excpt);
}
}
/**
* A structure grouping all options and additional information extracted from the first entry of the
* command-line parameters.
*/
private static class Options
{
/** Do the arguments respect the newer, simpler syntax? */
boolean newSyntax = true;
/** The command line contains explicit file names. */
boolean explicit = true;
/** The command line contains filespecs. */
boolean filespec = false;
/** The command line contains filespec plus an ignore-list. */
boolean ignore = false;
/** The optional database configuration profile. */
List<String> cfgProfile = new ArrayList<>();
/** Allow recursion in filespec mode or trail mode. */
boolean recursion = true;
/** The debug level. */
int debug = MSG_STATUS;
/** The index of the next parameter to be processed. */
int idx = 0;
}
}