TransformDriver.java
/*
** Module : TransformDriver.java
** Abstract : common code for transformation of a file or set of files
**
** Copyright (c) 2006-2023, Golden Code Development Corporation.
**
** -#- -I- --Date-- --------------------------------------Description---------------------------------------
** 001 GES 20181022 Extracted common code from ConversionDriver.
** 002 ECF 20190218 Added blacklist (filespec + ignore list) mode and dry-run mode (only list source files
** to be processed, but do nothing more).
** 003 CA 20190703 Legacy 4GL classes are always processed in BFS order by the conversion driver. They are
** added at the beginning of the list, with all other files in normal order, following them.
** 004 ECF 20190528 Ignore comments in ignore list (demarcated by a leading '#' character).
** 005 GES 20191011 Fix for processing Progress.Lang.Object.
** 006 CA 20200412 Added incremental conversion support.
** RFB 20200430 Fixed minor typo. Added some extra text to make added stand out.
** 007 CA 20200930 Force ANTLR to use Class.forName instead of loadClass.
** CA 20210213 Normalize the files read from the explicit or implicit file list, so that their names will
** use the OS separator. The explicit or implicit file list will always use the Linux-style
** '/' separator.
** CA 20210423 'tmpTabNames' has as value an AST which may be from a different file. For incremental
** conversion, if we include a file which affects values in this map, then all affected
** keys must be considered as dependencies and their files reconverted, too.
** GES 20210707 Removed some dead code and moved to use of some common logic in the FileList class.
** OM 20210923 Added -profile command line option for specifying the configuration profile.
** CA 20211012 Fixed generated WSDL and reworked SOAP support to rely on namespace, when resolving an
** operation.
** OM 20220325 Dynamic resolution of .dict, .schema and .p2o files as opposed to hardcoded path.
** OM 20220330 Moved conversion artifacts to ${cvtpath} folder.
** GES 20220320 Moved some of the inline file list processing back into new FileListFactory workers.
** CA 20220412 Added file-set support (either -Z command-line option or p2j.cfg.xml configuration).
** CA 20220419 The options are really optional, if no option is set, then only 'mode' is allowed and the
** p2j.cfg.xml is checked for a file-set (assuming 'Z' mode).
** TJD 20220504 Java 11 compatibility minor changes
** CA 20220501 Each profile has the same structure as the main 'global' config. Allow multiple profiles
** to be ran at once, with the conversion switching the state between profiles, when a
** resource (like a file or namespace) is being processed. Only front phase is supported at
** this time.
** CA 20220516 Made JobDefinition protected.
** CA 20220524 For incremental conversion, normalize the filenames, as this is how they are kept in the
** conversion database.
** GES 20210520 Sort interfaces into the BFS class hierachy so that all overridden methods will be
** converted in the correct order (the parent version first).
** 008 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 009 OM 20230115 Replaced absolutePath(), relativePath(), upPath() and downPath() with faster
** versions, based on node types rather on string paths.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
/*
TODOS:
- separate modes for various cleanup (deletion of different sets of output)
- more granular debug options (step specific instead of global)
*/
package com.goldencode.p2j.convert;
import java.io.*;
import java.util.*;
import java.util.logging.*;
import java.text.*;
import com.goldencode.ast.*;
import com.goldencode.util.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.db.*;
import com.goldencode.p2j.uast.*;
import com.goldencode.p2j.util.*;
import com.goldencode.p2j.pattern.*;
import com.goldencode.p2j.schema.*;
/**
* Provides a harness to drive the end-to-end transformation process for a user-defined list of files.
*/
public abstract class TransformDriver
{
/** No message output - quiet mode. */
public static final int MSG_NONE = 0;
/** Output status messages only. */
public static final int MSG_STATUS = 1;
/** Output debug and status messages. */
public static final int MSG_DEBUG = 2;
/** Full debug trace output. */
public static final int MSG_TRACE = 3;
/** Spacer for section headers and output. */
protected static final String spacer = StringHelper.repeatChar('-', 78);
/** Match list for database name conversion. */
protected static final String datanames = Configuration.getParameter("datanames");
/** Chars that must be escaped to eliminate regular expression meaning. */
protected static final char[] DANGER_LIST = { '{', '}', '[', ']', '(', ')' };
/** Form in which to obtain the list of files. */
protected static enum ListType { CMD_LINE, WHITELIST, BLACKLIST, FILESPEC, FILESET };
/** Logger */
private static final ConversionStatus LOG = ConversionStatus.get(TransformDriver.class);
static
{
try
{
String name = Configuration.getParameter("registry");
AstManager.initialize(() -> new XmlFilePlugin(name, Configuration.isImportMode()));
}
catch (AstException ae)
{
LOG.log(Level.SEVERE, "Conversion initialization failed!", ae);
}
}
/** The list of Progress source files (command line input). */
protected FileList source = null;
/** The list of Progress AST (abstract syntax tree) files. */
protected FileList asts = null;
/** The list of database dictionary files. */
protected FileList dicts = null;
/** The list of Progress source file specific dictionary files. */
protected FileList sdicts = null;
/** The debug level to honor throughout the process. */
protected int debug = MSG_STATUS;
/** The details of which features this job run must implement. */
protected RunMode mode = null;
/** Full details of this run. */
protected JobDefinition job = null;
/** Base class for all OO classes. */
private static final String ROOT_CLASS = "Progress.Lang.Object".toLowerCase();
/**
* Creates an instance with a specific configuration.
*
* @param job
* The details of the job to execute.
*/
public TransformDriver(JobDefinition job)
{
this.source = job.files;
this.mode = job.mode;
this.job = job;
if (job.debug < MSG_NONE || job.debug > MSG_TRACE)
{
throw new IllegalArgumentException("Invalid debug level: " + job.debug);
}
else
{
this.debug = job.debug;
}
}
/**
* Drive the front end of the conversion process using the file list
* of Progress source files that is already defined in this instance.
* <p>
* Once the scanning of the source files is complete, this processing will
* modify the files list to create one that exposes AST files instead of
* Progress source files.
* <p>
* This runs the following:
* <ul>
* <li> schema loader (optional)
* <li> preprocessor (optional)
* <li> lexer
* <li> parser
* <li> AST persister
* <li> schema fixups for the dictionary and for source-specific schema
* files (optional)
* <li> call graph generator (optional)
* <li> saves off a set results for enabling a future restore to this
* point
* </ul>
*
* @throws ConfigurationException If there is problem reading/parsing the configuration.
* @throws AstException If there is an problem processing ASTs.
* @throws SchemaException If there is a problem with the schema.
* @throws IOException If there is a problem with I/O.
* @throws Exception For any other problem.
*/
protected void front()
throws ConfigurationException,
AstException,
SchemaException,
IOException,
Exception
{
if (mode.schema)
{
// run schema loader to generate new dictionary files
runSchemaLoader();
}
// optionally preprocess, lex, parse and save the resulting ASTs as
// persisted XML files with the name sourcename.[pPwW].ast
runScanDriver(mode.preproc, mode.schema, job.abortOnError, job.threads);
// ensure that downstream processing has the proper list of files
convertSourceNamesToAstNames();
processTrees("Post-Parse Fixups",
"fixups/post_parse_fixups",
null,
false,
asts,
debug);
processTrees("Early Annotations",
"annotations/early_annotations",
null,
false,
asts,
debug);
processTrees("Gap Analysis Marking",
"gaps/gap_analysis_marking",
null,
false,
asts,
debug);
if (mode.schema)
{
// clean up previous run - build schema list and delete those files
FileList delDicts = createDatabaseList(".schema", false);
deleteFiles(delDicts.listFilenames());
// build dictionary list
dicts = createDatabaseList(".dict", omitSchemaMetadata());
// fixup dictionary files
processTrees("Schema Fixups (data dictionary)",
"schema/fixups",
datanames,
true,
dicts,
debug);
// build schema list
sdicts = convertSourceNamesToDictNames(".dict");
// fixup source file schema files (if any exist for these specific
// source files)
if (sdicts.size() > 0)
{
processTrees("Schema Fixups (Progress source file schemas)",
"schema/fixups",
datanames,
true,
sdicts,
debug);
}
}
if (mode.callgraph)
{
runCallGraphGenerator();
}
}
/**
* Runs the <code>SchemaLoader</code> class to convert <code>.df</code>
* files into an AST form which can be read by the
* <code>SchemaDictionary</code>. Files left by previous runs, if any,
* are removed first.
*
* @throws ConfigurationException If there is problem reading/parsing the configuration.
* @throws SchemaException If there is a problem with the schema.
* @throws IOException If there is a problem with I/O.
*/
protected void runSchemaLoader()
throws ConfigurationException,
SchemaException,
IOException
{
printHeader("SchemaLoader", debug);
// clean up previous run - build dictionary list and delete those files
FileList delDicts = createDatabaseList(".dict", false);
deleteFiles(delDicts.listFilenames());
SchemaLoader loader = new SchemaLoader();
if (debug == MSG_NONE)
{
loader.setQuietMode(true);
}
loader.importAll();
}
/**
* Runs the <code>ScanDriver</code> class to convert Progress source
* files into an AST form. This corresponds to preprocessing, lexing,
* parsing and AST persistence.
*
* @param preproc
* <code>true</code> if the <code>Preprocessor</code> should
* should be forced to run. <code>false</code> if previously
* cached preprocessor results should be honored.
* @param schema
* <code>true</code> if the <code>SchemaLoader</code> and
* <code>schema/fixups.rules</code> should be run. Note this
* is only to assist with cleaning files as the schema
* processing itself is not handled here.
* @param abortOnError
* <code>true</code> to end the scanning run if an error is encountered,
* else <code>false</code>.
* @param threads
* Number of threads to use for the scan. A non-positive value will enable
* the default behavior of a work-stealing thread pool with one thread per
* per available CPU. A positive value will create a fixed size thread pool.
*
* @throws Exception If there is a problem.
*/
protected void runScanDriver(boolean preproc,
boolean schema,
boolean abortOnError,
int threads)
throws Exception
{
printHeader("Scanning Progress Source (" +
(preproc ? "preprocessor, " : "") +
"lexer, parser, persist ASTs)",
debug);
long start = System.currentTimeMillis();
cleanScanResults(preproc, schema);
ScanDriver.scan(source, (debug == MSG_NONE), abortOnError, threads);
printElapsed(System.currentTimeMillis() - start);
}
/**
* Remove some or all scan driver output files associated with the
* current Progress source file list AND in the list of fakeout files
* that represent the built-in 4GL classes/interfaces or .NET classes
* that can be accessed. All output except preprocessor and schema output
* is unconditionally deleted.
*
* @param preproc
* <code>true</code> means that cached preprocessor files should
* be deleted.
* @param schema
* <code>true</code> means that schema related generated files
* should be deleted.
*
* @throws IOException If there is a problem with I/O.
*/
protected void cleanScanResults(boolean preproc, boolean schema)
throws IOException
{
ArrayList<String> exts = new ArrayList<>();
exts.add(".lexer");
exts.add(".parser");
exts.add(".ast");
exts.add(".jast");
if (preproc)
{
exts.add(".cache");
exts.add(".pphints");
}
if (schema)
{
exts.add(".dict");
exts.add(".schema");
exts.add(".schema.original");
exts.add(".p2o");
}
// we never delete ".schema.hints" or ".hints" as those are both
// hand-coded configuration files
String[] extslist = exts.toArray(new String[0]);
// do the work
zapFiles(source.listFilenames(), extslist);
if (!job.incremental)
{
zapFiles(SymbolResolver.listFakeoutFiles(), extslist);
}
}
/**
* Concatenate all extensions with all the files and if any resulting
* names exist, delete the matching file from the file system.
*
* @param files
* The list of file base names.
* @param extslist
* The list of extensions to zap.
*/
protected void zapFiles(String[] files, String[] extslist)
{
for (int i = 0; i < files.length; i++)
{
for (int j = 0; j < extslist.length; j++)
{
File check = new File(files[i] + extslist[j]);
if (check.exists() && check.isFile())
{
// delete the file
check.delete();
}
}
}
}
/**
* Process each input element and escape characters that would be
* interpreted as part of a regular expression to convert them all into
* a literal match. At this time, only a limited subset of characters
* is escaped (curly braces, square brackets and parenthesis). This allows
* valid file name characters to be passed literally in a larger file
* specification that will be interpreted as a regular expression.
*
* @param list
* List of literal strings that may need to be escaped.
*
* @return Safe version of the list.
*/
protected String[] escapeRegex(String[] list)
{
String[] result = new String[list.length];
for (int i = 0; i < list.length; i++)
{
StringBuilder sb = new StringBuilder();
int len = list[i].length();
for (int j = 0; j < len; j++)
{
char next = list[i].charAt(j);
for (int x = 0; x < DANGER_LIST.length; x++)
{
if (DANGER_LIST[x] == next)
{
sb.append('\\');
break;
}
}
sb.append(next);
}
result[i] = sb.toString();
}
return result;
}
/**
* Runs the <code>CallGraphGenerator</code> using the project's defined
* root nodes list to create a call graph for each root node.
*
* @throws ConfigurationException If there is problem reading/parsing the configuration.
* @throws AstException If there is an problem processing ASTs.
* @throws IOException If there is a problem with I/O.
*/
protected void runCallGraphGenerator()
throws ConfigurationException,
IOException,
AstException
{
printHeader("Generating Call Graphs", debug);
try
{
CallGraphGenerator.generateGraphs(null, debug);
}
finally
{
CallGraphHelper.shutdownGraph();
}
}
/**
* Run the pattern engine using the given rules filename as the program
* and the list of files are the ASTs (abstract syntax trees) to process.
*
* @param section
* The section name (for the header);
* @param rules
* The pattern engine program (known as a rules file) to run.
* @param matches
* The match list name to use or <code>null</code> to use the
* project default.
* @param db
* Load default database name mappings.
* @param files
* The list of ASTs to process.
* @param debug
* The debug level to use with the pattern engine.
*
* @return The list of 4GL program names generated by this engine run, reported by
* {@link PatternEngine#getNewProgramFiles()}.
*
* @throws ConfigurationException If there is problem reading/parsing the configuration.
* @throws AstException If there is an problem processing ASTs.
*/
protected static List<String> processTrees(String section,
String rules,
String matches,
boolean db,
FileList files,
int debug)
throws ConfigurationException,
AstException
{
printHeader(section, debug);
// load/reload a known match list for name conversion
NameConverter.load(matches, db);
// create a new pattern engine, configure it and run
PatternEngine engine = new PatternEngine();
engine.addAstSpec(files);
PatternEngine.setDebugLevel(debug);
engine.run(rules);
return engine.getNewProgramFiles();
}
/**
* Run the pattern engine using the given rules filename as the program
* and the list of files are the ASTs (abstract syntax trees) to process.
* <p>
* Also, set the variable initializers using the given map.
*
* @param section
* The section name (for the header);
* @param rules
* The pattern engine program (known as a rules file) to run.
* @param matches
* The match list name to use or <code>null</code> to use the
* project default.
* @param db
* Load default database name mappings.
* @param files
* The list of ASTs to process.
* @param debug
* The debug level to use with the pattern engine.
* @param vars
* The variable initializers.
*
* @throws ConfigurationException
* If there is problem reading/parsing the configuration.
* @throws AstException
* If there is a problem processing ASTs.
*/
protected void processTrees(String section,
String rules,
String matches,
boolean db,
FileList files,
int debug,
Map vars)
throws ConfigurationException,
AstException
{
printHeader(section, debug);
// load/reload a known match list for name conversion
NameConverter.load(matches, db);
// create a new pattern engine, configure it and run
PatternEngine engine = new PatternEngine();
engine.addAstSpec(files);
PatternEngine.setDebugLevel(debug);
engine.setVariableInitializers(vars);
engine.run(rules);
}
/**
* Delete the file associated with each name in the list.
*
* @param list
* List of explicit filenames to delete.
*
* @throws IOException
* If there is a problem with I/O.
*/
protected void deleteFiles(String[] list)
throws IOException
{
for (int i = 0; i < list.length; i++)
{
File next = new File(list[i]);
next.delete();
}
}
/**
* For each element in the spec list, generate a list of matching files
* (each spec may contain traditional file system wildcard characters)
* and delete each of these files. The given <code>ExcludeCheck</code>
* instance is used to remove results from the list on a callback basis.
*
* @param speclist
* List of file specifications.
* @param check
* If <code>null</code>, no callback occurs. Otherwise, the
* <code>exclude</code> method will be called on the passed instance.
*
* @throws IOException
* If there is a problem with I/O.
*/
protected void deleteFileSpecs(String[] speclist, ExcludeCheck check)
throws IOException
{
for (int i = 0; i < speclist.length; i++)
{
int idx = speclist[i].lastIndexOf(File.separator);
String dir = (idx < 0) ? "." : speclist[i].substring(0, idx);
String spec = speclist[i].substring(idx + 1);
// create a list of actual files matching this spec
FileList list = new FileSpecList(new File(dir), spec, false);
File[] fset = list.list();
for (int j = 0; j < fset.length; j++)
{
// prune out the preprocessor cache files
if (check != null && check.exclude(fset[j].getCanonicalPath()))
{
continue;
}
// delete the file
fset[j].delete();
}
}
}
/**
* Print a section header and optionally spacers, depending on debug
* level (if level > <code>MSG_NONE</code> then spacers are printed).
*
* @param section
* The section name.
* @param debug
* The debug level.
*/
protected static void printHeader(String section, int debug)
{
if (debug > MSG_NONE)
{
System.out.println();
System.out.println(spacer);
}
System.out.println(section);
if (debug > MSG_NONE)
{
System.out.println(spacer);
System.out.println();
}
}
/**
* Using the source name list as input, this method creates duplicate
* lists with the proper <code>.ast</code> or <code>.jast</code> file
* suffixes on each entry. The results are saved into the Progress AST
* name list.
*/
protected void convertSourceNamesToAstNames()
{
List<String> astFiles = new ArrayList<>();
astFiles.addAll(Arrays.asList(source.withSuffix(".ast")));
asts = new ExplicitFileList(astFiles.toArray(new String[0]));
Set<String> files = new LinkedHashSet<>();
// preserve the order as listed by FileList
files.addAll(Arrays.asList(asts.listFilenames()));
// get the legacy class hierarchy and create a BFS walk. these will be processed further
// in this order
List<String> classes = sortLegacyClasses(files);
List<String> sorted = new ArrayList<>(classes);
files.removeAll(classes);
sorted.addAll(files);
asts = new ExplicitFileList(sorted.toArray(new String[0]));
((ExplicitFileList) asts).setSort(false);
}
/**
* Scans the configuration file and create a list of artefacts containing a specific file for each declared
* namespace from the default profile, if such file exists.
* <br>
* While the extension is common for all databases, the location where the files are searched is specific
* to each database (namespace) and is computed using the {@code xmlFile} attribute of the
* {@code namespace} because all these artefacts are generated in same location.
* <br>
* By default, if no {@code xmlFile} attribute is specified for a namespace in the configuration, the
* project's {@code data/namespace} is assumed.
*
* @param extension
* The file extension to search for (usually {@code .dict}, {@code .schema}, or {@code .p2o}).
* @param omitMeta
* {@code true} to omit the metadata filename from the resulting list; {@code false} to retain it.
*
* @return The list of files found.
*
* @throws ConfigurationException
* If there is problem reading/parsing the configuration.
*/
protected FileList createDatabaseList(String extension, boolean omitMeta)
throws ConfigurationException
{
Configuration cfg = Configuration.getInstance();
SchemaConfig config = Configuration.getSchemaConfig();
String metaName = config.getMetadata().getName(); // default is "standard"
List<String> toReturn = new ArrayList<>();
Iterator<String> it = config.databases(cfg.isMultiProfile());
while (it.hasNext())
{
String schema = it.next();
if (cfg.isMultiProfile())
{
cfg.withDbProfile(schema, (profile) ->
{
String profileMetaName = config.getMetadata().getName();
// omit meta if so required
if (omitMeta && schema.equals(profileMetaName))
{
return;
}
String fileName = config.getSchemaFileName(schema, extension);
if (new File(fileName).exists() && !toReturn.contains(fileName))
{
// collect ONLY existing files
toReturn.add(fileName);
}
});
continue;
}
// omit meta if so required
if (omitMeta && schema.equals(metaName))
{
continue;
}
String fileName = config.getSchemaFileName(schema, extension);
if (new File(fileName).exists() && !toReturn.contains(fileName))
{
// collect ONLY existing files
toReturn.add(fileName);
}
}
return new ExplicitFileList(toReturn.toArray(new String[toReturn.size()]));
}
/**
* Determine whether we should omit the metadata schema when processing and persisting schema
* information.
*
* @return <code>true</code> if no metadata tables are specified by the project configuration,
* else <code>false</code>.
*/
protected boolean omitSchemaMetadata()
{
return Configuration.getSchemaConfig().getMetadata().getTables().isEmpty();
}
/**
* Using the source name list as input, this method creates a duplicate
* list with the given file suffix on each entry.
*
* @param suffix
* The file extension to search for (usually <code>.dict</code>
* or <code>.schema</code>).
*
* @return The list of files found.
*/
protected FileList convertSourceNamesToDictNames(String suffix)
{
return new ExplicitFileList(source.withSuffix(suffix));
}
/**
* Replace the filename extension with the given extension and return the new string. If the
* given filename already has the given extension, simply return the existing filename.
*
* @param filename
* File name; may not be <code>null</code>.
* @param append
* Extension to append after the filename's existing extension is removed. Must
* begin with the dot (<code>.</code>) character.
*
* @return Updated filename.
*/
protected String replaceFilenameExtension(String filename, String append)
{
if (filename.endsWith(append))
{
return filename;
}
StringBuilder buf = new StringBuilder();
int pos = filename.lastIndexOf(".");
if (pos < 0)
{
buf.append(filename);
}
else
{
buf.append(filename, 0, pos);
}
buf.append(append);
return buf.toString();
}
/**
* Handle the execution of the front, middle and back of the transformation process
* as defined in the job details.
*
* @param title
* Title for this driver's job.
*/
protected void executeJob(String title)
{
// set a global property to for Antlr to use Class.forName
System.setProperty("ANTLR_USE_DIRECT_CLASS_LOADING", "true");
// set the configuration profile
Configuration.setDefaultProfiles(job.cfgProfiles);
// capture the start time
long jobStartTime = System.currentTimeMillis();
try
{
// print the main header
printHeader(title, MSG_STATUS);
if (job.incremental)
{
printHeader("Incremental Conversion - checking file signature", MSG_STATUS);
ConversionData.connect();
// filter the files and keep only the following:
// - newly added files
// - changed files
String[] files = job.files.listFilenames();
for (int i = 0; i < files.length; i++)
{
files[i] = Configuration.normalizeFilename(files[i]);
}
Set<String> filteredFiles = new LinkedHashSet<>();
Set<String> dependencies = new LinkedHashSet<>();
dependencies.addAll(Arrays.asList(files));
Set<String> depFiles = new HashSet<>();
boolean force = false;
do
{
depFiles.clear();
for (String file : dependencies)
{
if (force || ConversionData.mustConvertFile(file))
{
System.out.println(file + " - added for conversion" + (force ? " (dependency)" : ""));
filteredFiles.add(file);
depFiles.addAll(ConversionData.clean(file));
}
else
{
System.out.println(file + " - unchanged");
}
}
depFiles.removeAll(filteredFiles);
dependencies.clear();
dependencies.addAll(depFiles);
filteredFiles.addAll(depFiles);
force = true;
}
while (!depFiles.isEmpty());
ConversionData.disconnect();
if (filteredFiles.isEmpty())
{
System.out.println("INCREMENTAL CONVERSION: no changed files found");
return;
}
job.files = new ExplicitFileList(filteredFiles.toArray(new String[0]),
job.files.isCaseSensitive());
source = job.files;
}
else
{
ConversionData.cleanAll();
}
// front-end
if (mode.front)
{
front();
}
// middle
if (mode.middle)
{
middle();
}
// back-end
if (mode.back)
{
back();
}
}
catch (Exception exc)
{
LOG.log(Level.SEVERE, "ERROR:", exc);
System.exit(1);
}
finally
{
ConversionData.disconnect();
// print out total time
printElapsed(System.currentTimeMillis() - jobStartTime);
}
}
/**
* Drive the middle of the transformation process using the file list of Progress AST files
* that is already defined in this instance.
*
* @throws ConfigurationException If there is problem reading/parsing the configuration.
* @throws AstException If there is an problem processing ASTs.
* @throws SchemaException If there is a problem with the schema.
* @throws IOException If there is a problem with I/O.
*/
protected abstract void middle()
throws ConfigurationException,
AstException,
SchemaException,
IOException;
/**
* Drive the back end of the transformation process using the file list of Progress AST files
* that is already defined in this instance.
*
* @throws ConfigurationException If there is problem reading/parsing the configuration.
* @throws AstException If there is an problem processing ASTs.
* @throws SchemaException If there is a problem with the schema.
* @throws IOException If there is a problem with I/O.
*/
protected abstract void back()
throws ConfigurationException,
AstException,
SchemaException,
IOException;
/**
* Enables delegation of list processing to another method while providing
* a callback to determine if a given list entry needs to be excluded from
* processing.
*/
protected abstract class ExcludeCheck
{
/**
* Determines if a given name should be excluded from list processing.
*
* @param name
* The list element to check.
*
* @return <code>true</code> if the element should be excluded from
* list processing.
*/
abstract boolean exclude(String name);
}
/**
* Helper to only exclude the generated preprocessor files (cache and
* preprocessor hints) conditionally and to always exclude the manually
* coded hints files.
*/
protected class ScanCheck
extends ExcludeCheck
{
/** <code>true</code> if preprocessor files should be deleted. */
protected boolean preproc = false;
/** <code>true</code> if schema files should be deleted. */
protected boolean schema = false;
/**
* Construct and instance and store the preprocessor delete setting.
*
* @param preproc
* <code>true</code> if preprocessor files should be deleted.
* @param schema
* <code>true</code> if schema files should be deleted.
*/
ScanCheck(boolean preproc, boolean schema)
{
this.preproc = preproc;
this.schema = schema;
}
/**
* Determines if a given name should be excluded from list processing.
*
* @param name
* The list element to check.
*
* @return <code>true</code> if the element should be excluded from
* list processing.
*/
boolean exclude(String name)
{
return (!preproc && name.endsWith(".cache")) ||
(!preproc && name.endsWith(".pphints")) ||
(!schema && name.endsWith(".dict")) ||
(!schema && name.endsWith(".schema")) ||
(!schema && name.endsWith(".schema.original")) ||
(!schema && name.endsWith(".p2o")) ||
name.endsWith(".schema.hints") ||
name.endsWith(".hints");
}
}
/**
* Interpret the mode command line argument and convert this into the
* effective run mode object. Multiple "+" delimited mode values can
* be included in the same argument. In this case, the result is the
* logical OR of all the activated flags in each specified mode.
*
* @param key
* The mode command line argument.
* @param modes
* Valid run modes.
* @param help
* Help text for syntax display.
*
* @return The run mode to use for controlling program flow.
*/
protected static RunMode interpretRunMode(String key,
Map<String, RunMode> modes,
String[] help)
{
String[] cand = key.split("\\+");
RunMode result = null;
if (key.length() == 0 || cand.length == 0)
{
syntax(help, "Missing mode parameter.", -4);
}
for (int i = 0; i < cand.length; i++)
{
RunMode next = modes.get(cand[i]);
if (next == null)
{
syntax(help, "Unrecognized mode '" + key + "'.", -5);
}
if (result == null)
{
// our first match sets the starting result
result = next;
}
else
{
// logically OR these together so that the result is the
// combination of all true flags
result = result.merge(next);
}
}
return result;
}
/**
* Print out a simple elapsed time counter.
*
* @param elapsed
* Number of milliseconds of time elapsed.
*/
protected static void printElapsed(long elapsed)
{
Date date = new Date(elapsed - TimeZone.getDefault().getRawOffset());
SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");
System.out.println();
System.out.println(spacer);
System.out.println("Elapsed job time: " + format.format(date));
System.out.println();
}
/**
* Emit a syntax statement and optional error message to <code>stderr</code>, then exit the
* process with the specified return code. This method is called when the user passes command
* line parameters which are not viable.
*
* @param helpText
* Help text for syntax display.
* @param msg
* Optional error message; may be <code>null</code>.
* @param rc
* Process return code used with <code>System.exit()</code>.
*/
protected static void syntax(String[] helpText, String msg, int rc)
{
if (msg != null)
{
LOG.log(Level.SEVERE, msg);
}
if (helpText != null)
{
LOG.log(Level.WARNING, String.join(System.lineSeparator(), helpText));
}
System.exit(rc);
}
/**
* Container for the details of a driver run.
*/
protected static class JobDefinition
{
/** Flag indicating if the incremental conversion mode is on. */
public final boolean incremental;
/** 4GL source files to process. */
public /*final*/ FileList files;
/** Debug level. */
public final int debug;
/** Defines the features of this job. */
public final RunMode mode;
/** Abort on error flag. */
public final boolean abortOnError;
/**
* Number of threads to use for the scan. A non-positive value will enable the default behavior of a
* work-stealing thread pool with one thread per available CPU. A positive value will create a fixed
* size thread pool.
*/
public final int threads;
/** User-defined rule-set. */
public final String transform;
/** The optional configuration profile(s). */
public final List<String> cfgProfiles;
/**
* Defines a new job.
*
* @param run
* Defines the features of this job.
* @param transform
* User-defined rule-set.
* @param files
* The 4GL source files to process.
* @param cfgProfiles
* The optional configuration profiles.
* @param threads
* Number of threads to use for the scan.
* @param abortOnError
* Abort on error flag.
* @param incremental
* Flag indicating if the incremental conversion mode is on.
* @param debug
* The debug level.
*/
protected JobDefinition(RunMode run,
String transform,
FileList files,
List<String> cfgProfiles,
int threads,
boolean abortOnError,
boolean incremental,
int debug)
{
this.mode = run;
this.transform = transform;
this.files = files;
this.cfgProfiles = cfgProfiles == null || cfgProfiles.isEmpty() ? null : new ArrayList<>(cfgProfiles);
this.threads = threads;
this.abortOnError = abortOnError;
this.incremental = incremental;
this.debug = debug;
}
}
/**
* Process the command line interface for the driver.
* <p>
* The program will return a non-zero code if there is a fatal error during the transformation processing.
* Each child class implements its own command line syntax, but the options, modes and 4GL source file
* lists are handled in a common way. The mode names are driver- specific, however the various front end
* modes are common since all drivers need to be able to create ASTs.
* <p>
* Syntax:
* <pre>{@code java -DP2J_HOME=<home> <TransformDriverChildClass> [-options]* <mode> [user_defined_rule_set] <file-specs>}</pre>
* Where:
* <ul>
* <li>Options (F, S, and X are mutually exclusive. If none is specified <file-specs> is a list of files
* added to command line):
* <ul>
* <li>Dn = set debug level 'n' (must be a numeric digit between {@code MSG_NONE} (0) and
* {@code MSG_TRACE} (3) inclusive
* <li>I = incremental conversion; only changed/new programs will be parsed and converted.
* <li>F = {@code <file-specs>} is a {@code <filename>} containing a custom FILE-LIST, instead of
* using the explicit command line file list
* <li>S = {@code <file-specs>} is composed of a root {@code <directory>} and a {@code
* "<file-filter>"}, to select SPECIFIC files instead of an explicit command line file list
* <li>X = {@code <file-specs>} is similar to S but has an additional custom EXCLUDE list file
* {@code "<filename>"} to be ignored
* <li>N = NO recursion in {@code <file-filter>} (S/X options)
* <li>Z = {@code <file-set>} is a {@code <filename>} containing the definition file list and
* Include plus eXclude operations to be applied on it
* <li>A = ABORT scan part of front end on an error
* <li>Tn = specify number of THREADS 'n' to use for front end file scan ('n' must be {@code >=}
* 0; defaults to 1; set to 0 to use 1 thread per available CPU)
* WARNING: this mode is not compatible if there are legacy .cls files
* <li>L = LIST files to be processed only; do not perform any further task
* <li>Pname = {@code <name>} of the configuration PROFILE follows the option. Can appear
* multiple times, to specify which profiles are included in this run.
* </ul>
* <li>{@code mode} (one or more of the following values, use '+' to delimit if there are two or more
* values; don't insert spaces!)
* <li>The optional {@code user_defined_rule_set} specifies the user-defined rule-set filename to execute
* (only some modes support this, see sub-class documentation for details)
* <li>explicit command line list (default approach if none of S, P, or F are present)
* <ul>
* <li>{@code <filelist>} is arbitrary list of absolute and/or relative file names to scan, this
* list is hard coded in the command line itself
* <li>any number of files may be listed on the command line, subject to command shell limits
* </ul>
* <li>custom file list (FILE approach)
* <ul>
* <li>{@code <filename>} is a single absolute or relative filename of a text file that contains a
* custom list of absolute and/or relative file names of files to scan
* <li>in other words, the file list will be read from a file instead of being hard coded on the
* command line
* <li>there must be one filename per line in the file
* <li>there is no limit of the number of files in the list
* </ul>
* <li>file-spec mode (expressly SPECIFIED files approach)
* <ul>
* <li>{@code <directory>} is the root directory to search for files
* <li>{@code "filespec"} is the file specification that will be used to create a list of files to
* process, should be enclosed in double quotes if any of the wildcard characters '*' or '?'
* are used
* </ul>
* If no files are specified, then the {@code <file-set>} configuration from {@code p2j.cfg.xml} will
* be used (the Z mode is used implicitly). If there is no file-set, then the program stops.
* <li>custom ignore list (file-spec mode with additional EXCLUDE list)
* <ul>
* <li>{@code <directory>} is the root directory to search for files
* <li>{@code "filespec"}" is the file specification that will be used to create a list of files
* to process, should be enclosed in double quotes if any of the wildcard characters '*' or
* '?' are used
* <li>{@code <filename>} is a single absolute or relative filename of a text file that contains a
* custom list of specifications of file names to ignore
* <li>asterisk (*) and question mark (?) wildcards are permitted to represent zero or more and
* exactly one wildcard characters, respectively
* <li>files specified by the primary {@code filespec} which are not matched by the ignore list
* are scanned as they would be in {@code filespec} mode.
* </ul>
* </ul>
*
* @param args
* List of command line arguments.
* @param modes
* Valid run modes.
* @param help
* Help text for syntax display.
*
* @return The details of this job.
*/
protected static JobDefinition processCommandLine(String[] args, Map<String, RunMode> modes, String[] help)
{
// option defaults
ListType type = ListType.CMD_LINE;
boolean recursion = true;
boolean abortOnError = false;
boolean dryRun = false;
int debug = MSG_STATUS;
int idx = 0;
int threads = 1;
boolean incremental = false;
List<String> cfgProfiles = new ArrayList<>();
// requires (mode + directory + filespec) OR (mode and a filelist) OR (mode), with the file-set read from
// p2j.cfg.xml
if (args.length < 1)
{
syntax(help, null, -1);
}
// process options
while (args[idx].startsWith("-"))
{
String opts = args[idx].substring(1).toLowerCase(); // drop the '-'
idx++;
if (opts.startsWith("p"))
{
// this must be a single option token because the configuration PROFILE name might contain
// characters which interfere with other options
cfgProfiles.add(opts.substring(1)); // drop the 'p'
continue;
}
// the next options might all be concatenated after a single "-" prefix but this practice is
// obsolete and not recommended. Use each option independently.
if (opts.indexOf('i') != -1)
{
incremental = true;
}
if (opts.indexOf('s') != -1)
{
type = ListType.FILESPEC;
}
if (opts.indexOf('f') != -1)
{
type = ListType.WHITELIST;
}
if (opts.indexOf('x') != -1)
{
type = ListType.BLACKLIST;
}
if (opts.indexOf('z') != -1)
{
type = ListType.FILESET;
}
if (opts.indexOf('n') != -1)
{
recursion = false;
}
if (opts.indexOf('a') != -1)
{
abortOnError = true;
}
if (opts.indexOf('l') != -1)
{
dryRun = true;
}
int dbg = opts.indexOf('d');
if (dbg != -1)
{
// get level (set to space char if of out-of-bounds to avoid exception)
char level = (dbg + 1 < opts.length() ? opts.charAt(dbg + 1) : ' ');
if (level == ' ' || !Character.isDigit(level))
{
syntax(help, "Missing numeric debug level character following D option.", -2);
}
else
{
debug = Character.digit(level, 10);
if (debug < MSG_NONE || debug > MSG_TRACE)
{
syntax(help, "Debug level must be between " + MSG_NONE + " and " + MSG_TRACE + ".", -3);
}
}
}
int tpos = opts.indexOf('t');
if (tpos != -1)
{
int start = tpos + 1;
int next = start;
int len = opts.length();
while (next < len && Character.isDigit(opts.charAt(next)))
{
next++;
}
boolean fail = next == start;
if (!fail)
{
try
{
threads = Integer.parseInt(opts.substring(start, next));
}
catch (NumberFormatException exc)
{
// should not be thrown, since we pre-validated characters as digits
fail = true;
}
}
if (fail)
{
syntax(help, "Missing number of threads following T option.", -4);
}
}
}
// process the mode argument
String key = args[idx].toUpperCase();
RunMode run = interpretRunMode(key, modes, help);
idx++;
String transform = null;
// user-defined rule-set (TODO: hard coding here is should be replaced with a more generic approach)
if (run.middle && key.contains("TR"))
{
if (args.length <= idx)
{
syntax(help, "Missing user-defined rule-set parameter.", -6);
}
transform = args[idx];
idx++;
}
// configuration profiles (without schema) need to be loaded now, as the file-sets are required to be
// known bellow.
Configuration.loadConfigProfiles(cfgProfiles);
FileList files = null;
// at this point idx points to the next arg after the options, this
// must be one of our various forms of filename input
switch (type)
{
case CMD_LINE:
if (args.length <= idx && Configuration.getFileList() != null)
{
// no explicit command-line files are provided and p2j.cfg.xml has configured file-set
files = Configuration.getFileList();
}
if (files == null)
{
// there must be at least 1 arg left
if (args.length <= idx)
{
syntax(help, "Missing file list parameter.", -7);
}
int len = args.length;
String[] filelist = new String[len - idx];
if (len - idx >= 0)
{
System.arraycopy(args, idx, filelist, 0, len - idx);
}
files = new ExplicitFileList(filelist);
}
break;
case WHITELIST:
// there must be exactly 1 arg left
if ((args.length - idx) != 1)
{
syntax(help, "Missing custom file list parameter.", -8);
}
files = FileListFactory.createExplicit(args[idx]);
break;
case FILESPEC:
// there must be exactly 2 args left
if ((args.length - idx) != 2)
{
syntax(help, "Missing/incomplete filespec parameter.", -9);
}
files = new FileSpecList(new File(args[idx]), args[idx + 1], recursion);
break;
case BLACKLIST:
// there must be exactly 3 args left: starting directory, filespec, ignore filename
if ((args.length - idx) != 3)
{
syntax(help, "Missing/incomplete filespec and exclude list parameters.", -10);
}
// TODO: we could set case-sensitivity from the legacy system setting, which would allow us to
// read blacklists made with inputs on case-insensitive systems
files = FileListFactory.createBlacklist(args[idx],
args[idx + 1],
recursion,
false,
args[idx + 2]);
break;
case FILESET:
if ((args.length - idx) != 1)
{
syntax(help, "Missing file set parameter.", -11);
}
FileSet fileSet = FileListFactory.loadFileSet(args[idx]);
if (fileSet != null)
{
try
{
files = FileListFactory.processFileSet(fileSet, Configuration.isCaseSensitive());
}
catch (IOException e)
{
LOG.log(Level.SEVERE, "Processing file set:", e);
return null;
}
}
break;
}
if (files == null)
{
return null;
}
// if -L option was used, just list files which would be processed to stdout, return null
if (dryRun)
{
LOG.log(Level.CONFIG, "Dry run; listing source files only...");
for (File file : files.list())
{
LOG.log(Level.CONFIG, file.toString());
}
return null;
}
return new JobDefinition(run, transform, files, cfgProfiles, threads, abortOnError, incremental, debug);
}
/**
* Read the class hierarchy from the AST associated with the given filename and store the state in
* the given maps.
*
* @param mgr
* AST manager.
* @param cls2file
* Map of qualified class names to file names.
* @param clsHierarchy
* Map of qualified class names to a set of subclasses.
* @param file
* The file name to process.
*/
private static void processOODefinition(AstManager mgr,
Map<String, String> cls2file,
Map<String, Set<String>> clsHierarchy,
String file)
{
// temporarily load this AST and get:
// - the qualified class name
// - the super class (single inherits for a class) or super interfaces (multiple-inherits for
// interface)
// - all implemented interfaces (which we treat the same as a superclass) for a class
Aast ast = mgr.loadTree(file);
Aast def = ast.getImmediateChild(ProgressParserTokenTypes.CLASS_DEF, null);
Aast header = null;
if (def == null)
{
def = ast.getImmediateChild(ProgressParserTokenTypes.INTERFACE_DEF, null);
if (def == null)
{
// must be an enum, nothing to do
return;
}
else
{
header = def.getImmediateChild(ProgressParserTokenTypes.KW_INTERFAC, null);
}
}
else
{
header = def.getImmediateChild(ProgressParserTokenTypes.KW_CLASS, null);
}
List<String> supers = new LinkedList<String>();
if (header.downPath(ProgressParserTokenTypes.KW_INHERITS, ProgressParserTokenTypes.CLASS_NAME))
{
Aast ref = header.getImmediateChild(ProgressParserTokenTypes.KW_INHERITS, null);
processClassNames(ref, clsHierarchy, supers);
}
if (header.downPath(ProgressParserTokenTypes.KW_IMPLEMTS, ProgressParserTokenTypes.CLASS_NAME))
{
Aast ref2 = header.getImmediateChild(ProgressParserTokenTypes.KW_IMPLEMTS, null);
processClassNames(ref2, clsHierarchy, supers);
}
if (supers.isEmpty())
{
supers.add(ROOT_CLASS);
}
Aast sym = header.getImmediateChild(ProgressParserTokenTypes.SYMBOL, null);
String qname = sym.getText().toLowerCase();
cls2file.put(qname, file);
// link the qname into the set for every super class/super interface/implemented interface
for (String superClass : supers)
{
Set<String> subClasses = clsHierarchy.get(superClass);
// create a new set if the superclass hasn't been processed yet
if (subClasses == null)
{
clsHierarchy.put(superClass, subClasses = new TreeSet<>());
}
// Progress.Lang.Object will recurse endlessly if we try to make it a subclass of itself
if (!superClass.toLowerCase().equals(qname))
{
subClasses.add(qname);
}
}
}
/**
* Process the given AST node for all {@code CLASS_NAME} child nodes. For each child node, read
* the qualified class name and add it to the given list of super classes. For built-in classes
* link them to the root class even if they are not direct children.
*
* @param ref
* The {@code KW_INHERITS} or {@code KW_IMPLEMTS} node which has one or more class name
* children.
* @param clsHierarchy
* The map of subclasses for each parent class.
* @param supers
* The list to which to add any inherited super class, inherited interfaces and any
* implemented interfaces.
*/
private static void processClassNames(Aast ref,
Map<String, Set<String>> clsHierarchy,
List<String> supers)
{
Aast child = ref.getImmediateChild(ProgressParserTokenTypes.CLASS_NAME, null);
while (child != null)
{
Boolean builtinCls = (Boolean) child.getAnnotation("builtin-cls");
String superClass = (String) child.getAnnotation("qualified");
superClass = superClass.toLowerCase();
supers.add(superClass);
if (builtinCls != null && builtinCls.booleanValue() && !ROOT_CLASS.equals(superClass))
{
// for all builtin classes, add them to the 'progress.lang.object' sub-classes
Set<String> subClasses = clsHierarchy.get(ROOT_CLASS);
if (subClasses == null)
{
clsHierarchy.put(ROOT_CLASS, subClasses = new TreeSet<>());
}
subClasses.add(superClass);
}
child = child.getImmediateChild(ProgressParserTokenTypes.CLASS_NAME, child);
}
}
/**
* Sort all legacy classes in the specified sources, by computed the class hierarchy and
* performing a BFS walk starting from <code>Progress.Lang.Object</code>. Interfaces are not
* included in this sort.
*
* @param sources
* The sources to sort.
*
* @return The sources, where classes are at the beginning of the list, in BFS order, with
* the rest of the files following it, in alphabetical order.
*/
private static List<String> sortLegacyClasses(Set<String> sources)
{
AstManager mgr = AstManager.get();
Map<String, String> cls2file = new HashMap<>();
Map<String, Set<String>> clsHierarchy = new HashMap<>();
for (String file : sources)
{
if (file.endsWith(".cls.ast"))
{
processOODefinition(mgr, cls2file, clsHierarchy, file);
}
}
List<String> bfs = walkHierarchy(clsHierarchy, ROOT_CLASS);
List<String> files = new ArrayList<>();
for (String cls : bfs)
{
String file = cls2file.get(cls);
if (file != null)
{
// may be null in case of skeleton classes
files.add(file);
}
}
return files;
}
/**
* Perform a BFS walk of the class hierarchy.
*
* @param hierarchy
* A map of classes to their direct sub-classes.
* @param parent
* The root class to start the walk.
*
* @return The class list, in BFS order.
*/
private static List<String> walkHierarchy(Map<String, Set<String>> hierarchy, String parent)
{
parent = parent.toLowerCase();
Set<String> subClasses = hierarchy.get(parent);
if (subClasses == null)
{
return Collections.emptyList();
}
List<String> res = new ArrayList<>();
res.addAll(subClasses);
for (String subCls : subClasses)
{
res.addAll(walkHierarchy(hierarchy, subCls));
}
return res;
}
}