ScanDriver.java
/*
** Module : ScanDriver.java
** Abstract : processes an entire set of files using the AstGenerator
**
** Copyright (c) 2004-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ----------------------------Description-----------------------------
** 001 GES 20050126 @19448 First version, with support for scanning
** a list of files based on user input. Each
** file in the entire subdirectory tree that
** matches the user's file spec will be
** preprocessed (if a cached version of the
** preprocessor output doesn't already exist)
** and then parsed. The parse output will
** be stored in a filename with the same base
** as the input file + ".parse". This is just
** a harness to automate scanning of sets of
** files.
** 002 GES 20050218 @19859 Added a try/catch inside the main processing
** loop. This allows for any exception on a
** given source file to be caught and the
** scan to continue. This way a long-running
** scan won't abort in the middle requiring
** a restart of the entire scan.
** 003 GES 20050221 @19909 Added an option to control whether the
** driver creates lexer output, parser output
** and/or a persistent AST.
** 004 GES 20050224 @19947 Moved parser output processing and AST
** persistence processing into AstGenerator.
** This class is now only a command line
** driver to run the AstGenerator on a user
** specified list of source files AND with
** user-specified control over generator
** options.
** 005 GES 20050228 @20105 At the end of scanning, trigger a registry
** save to ensure that any changes are
** persisted. Added a mode to rebuild the
** registry from persisted ASTs.
** 006 GES 20060126 @24084 Added options for no recursion, silent mode
** and an explicit file list mode.
** 007 GES 20070905 @34950 Modified the signature for processing files
** to enable the file list to be maintained
** downstream. This avoids some duplicate
** processing when a file shows up as an E4GL
** (HTML file) and its generated target file is
** also in the file list. The generated target
** file name must be removed from the list but
** the scan driver does not know enough to do
** that task. So it is delegated to the AST
** generator.
** 008 GES 20090429 @42064 Match package and class name changes.
** 009 GES 20090515 @42229 Moved to AstManager from AstRegistry/AstPersister.
** 010 ECF 20160302 Enabled multiple scanning threads. Files may be processed out of
** order, but this is safe. Also added an option to abort immediately on
** error, but the default behavior is to continue processing on error.
** Note that many errors (especially those encountered by the
** SymbolResolver) are caught at a lower level and are not rethrown to
** this level; these of course will not abort the scan.
** 011 ECF 20170213 When scanning with 1 thread, print name of file being scanned before
** processing it. This guarantees the filename appears before any
** messages about its processing.
** 012 ECF 20170302 Fixed remaining single threaded ordering issue.
** 013 GES 20170708 Pass silent flag down to the AST generator.
** 014 OM 20220405 XmlFilePlugin can load resources from application's jar.
** CA 20220501 Each profile has the same structure as the main 'global' config. Allow multiple
** profiles to be ran at once, with the conversion switching the state between
** profiles, when a resource (like a file or namespace) is being processed. Only
** front phase is supported at this time.
** GES 20220516 Avoid duplicate parsing caused by recursive parsing of classes.
** CA 20220727 Improved memory management for parsing; cleanup is done in two phases:
** 1. after each legacy class file has finished parsing, the SchemaDictionary will
** keep only protected temp-tables (all private tables are removed, as these
** can't be reached from sub-classes; all permanent tables are removed, as each
** class has its own reference to the permanent tables). Also, the class def
** instance will reduce its own used memory, which is not required when parsing
** sub-classes.
** 2. after parsing of the entire file set is finished, any SchemaDictionary or
** other ASTs referenced by the ClassDefinition are released.
** 015 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 016 CA 20230801 Changed logging level from FINE to SEVERE for the 'Failure in file' stacktraces.
** 017 CA 20241118 Mark already parsed .cls file's ASTs as 'original' in the registry.
** 018 ES 20250327 Added sources from conversion list to SymbolResolver.
** ES 20250328 Added in conversion list the referenced sources which are not present in the
** list.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.uast;
import java.io.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.logging.*;
import com.goldencode.ast.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.util.*;
/**
* Provides a simple harness to scan a user-defined list of files and drive
* preprocessing and parsing of each file. Uses {@link AstGenerator} for
* the majority of its function. Preprocessing output will be cached and
* reused as long as the cache file is newer than the input file. This
* significantly reduces the runtime of a multi-file scan. All files are
* parsed each time (there is no reuse of parser output) and the output
* is saved into one of more possible outputs (based on user controlled
* options):
* <p>
* <ul>
* <li> the preprocessor can be invoked or bypassed
* <li> the preprocessor output can be cached for future use
* <li> the output of the lexer in a file that matches the base
* filename of the input file plus an appended '.lexer'.
* <li> a human readable report with a filename that matches the base
* filename of the input file plus an appended '.parse'.
* <li> a persisted AST that can be reconstituted for later use without
* re-running the lexer and parser, with a filename that matches the
* base filename of the input file plus an appended '.ast'.
* </ul>
* <p>
* This program provides a command line interface. See {@link #main} for the
* syntax.
* <p>
* The scan is performed across multiple threads to reduce execution time. The API
* allows for a fixed number of threads to be specified. If 0 threads are specified,
* the scan driver will use one thread per available processor, as reported by
* <code>Runtime.getRuntime().availableProcessors()</code>.
* <p>
* <strong>Important Note Regarding Multi-threading</strong><br>
* In order to preserve the order of filenames written to the console, the writing of these names
* is deferred until all the processing of a file, and of all those files submitted for processing
* before it, has finished. This preserves the order of output of those filenames. However, since
* most errors and warnings are written to stderr directly from the code which handles them, error
* and warning information will display to the console <em>before</em> the name of the affected
* file.
* <p>
* When running with a thread pool of size 1, the above does not apply. In this case, we enter a
* stricter mode which forces the filename to appear before any messages associated with it.
* This mode is particularly useful in the early stages of a conversion project, when working
* through unstable or unknown code which is likely to produce errors or warnings. Using a single
* thread naturally preserves the original order of the filenames.
* <p>
* When running with a thread pool larger than 1, the filename will appear below any messages as
* noted above, but not necessarily immediately below. Several filenames and/or messages may be
* written before the filename associated with the messages. Furthermore, the messages themselves
* are not guaranteed to be written completely to stderr from one thread before another thread
* interrupts with messages of its own. Therefore, it is best to use multiple threads only once
* a source code set has stabilized and no longer produces errors and warnings during the scan
* phase.
* <p>
* TODO: to fix this, we must overhaul the way error handling is performed by the classes invoked
* by this class.
*/
public class ScanDriver
{
/** Logger. */
private static final ConversionStatus LOG = ConversionStatus.get(ScanDriver.class);
/** Keeps track of the class of the files referenced in a source from conversion list. */
private static Set<File> additionalFileList = Collections.synchronizedSet(new HashSet<>());
/** Keeps track of the class of the files referenced in a source from conversion list. */
private static Set<File> removedConversionFiles = Collections.synchronizedSet(new HashSet<>());
/** Iterator over the conversion list. */
private static ListIterator<File> iter = null;
static
{
try
{
String name = Configuration.getParameter("registry");
AstManager.initialize(new XmlFilePlugin(name, Configuration.isImportMode()));
}
catch (AstException ae)
{
// TODO: log this? get ready for null pointer exceptions later
}
}
/**
* Add to additionalFileList and to conversion list iterator a class referenced in of
* sources from conversion list.
*
* @param file
* Referenced 4gl source.
*/
public static void addSource(File file)
{
if (!additionalFileList.contains(file))
{
additionalFileList.add(file);
synchronized (iter)
{
iter.add(file);
iter.previous();
}
}
}
/**
* Remove from the conversion a buildin 4GL file.
*
* @param file
* Referenced 4gl source.
*/
public static boolean removeSource(File file)
{
return removedConversionFiles.add(file);
}
/**
* Drive the scanning process for each of the given files in the list,
* with preprocessing activated and caching of all output (preprocessor,
* lexer, parser), persistence of ASTs and no registry rebuild.
*
* @param files
* The list of files. Must not be <code>null</code>.
* @param silent
* <code>false</code> if screen (stdout) output should be done.
* If <code>true</code>, normal screen output is suppressed and
* the only output occurs on failures (this is written to
* stderr).
*
* @throws Exception
* in the event <code>abortOnError</code> is true and a failure occurs.
*/
public static void scan(FileList files, boolean silent)
throws Exception
{
// scan with default thread pool
scan(files, silent, false, 1);
}
/**
* Drive the scanning process for each of the given files in the list,
* with preprocessing activated and caching of all output (preprocessor,
* lexer, parser), persistence of ASTs and no registry rebuild.
*
* @param files
* The list of files. Must not be <code>null</code>.
* @param silent
* <code>false</code> if screen (stdout) output should be done.
* If <code>true</code>, normal screen output is suppressed and
* the only output occurs on failures (this is written to
* stderr).
* @param abortOnError
* <code>true</code> to end the scanning run if an error is encountered,
* else <code>false</code>.
* @param threads
* Number of threads to use for the scan. A non-positive value will enable
* the default behavior of a fixed size thread pool with one thread per
* available CPU. A positive value will create a fixed size thread pool of
* the given size.
*
* @throws Exception
* in the event <code>abortOnError</code> is true and a failure occurs.
*/
public static void scan(FileList files, boolean silent, boolean abortOnError, int threads)
throws Exception
{
scan(files, true, true, true, true, true, silent, abortOnError, threads);
}
/**
* Drive the scanning process for each of the given files in the list,
* honoring the caching and output options provided. In every case, the
* lexer and parser will be run. However there is control provided over
* whether or not the preprocessor runs. There is also very granular
* control provided over output files and screen output.
*
* @param files
* The list of files. Must not be <code>null</code>.
* @param preproc
* <code>true</code> if the preprocessor should be run.
* @param cache
* <code>true</code> if the preprocessor output should be saved persistently.
* @param lex
* <code>true</code> if the lexer output should be saved persistently.
* @param parse
* <code>true</code> if the parser output should be saved persistently.
* @param persist
* <code>true</code> if the resulting AST files should be saved persistently.
* @param silent
* <code>false</code> if screen (stdout) output should be done.
* If <code>true</code>, normal screen output is suppressed and
* the only output occurs on failures (this is written to stderr).
* @param abortOnError
* <code>true</code> to end the scanning run if an error is encountered,
* else <code>false</code>.
* @param threads
* Number of threads to use for the scan. A value of zero will enable the
* default behavior of a fixed size thread pool with one thread per available
* CPU. A positive value will create a fixed size thread pool of the given
* size.
*
* @throws Exception
* in the event <code>abortOnError</code> is true and a failure occurs.
*/
public static void scan(FileList files,
boolean preproc,
boolean cache,
boolean lex,
boolean parse,
boolean persist,
boolean silent,
boolean abortOnError,
int threads)
throws Exception
{
// setup
File[] list = files.list();
Set<String> fileCvtList = new HashSet<>();
for (int i = 0; i < list.length; i++)
{
try
{
fileCvtList.add(list[i].getCanonicalPath());
}
catch (IOException exc)
{
LOG.log(Level.SEVERE, "", exc);
}
}
SymbolResolver.setCvtSources(fileCvtList);
// reorganize files as necessary to minimize redundant HTML processing
list = AstGenerator.prioritize(list);
// create a thread-safe list of files
List<File> safeList = Collections.synchronizedList(new ArrayList<>(Arrays.asList(list)));
// use all available processors if a hard thread limit was not specified
if (threads == 0)
{
threads = Runtime.getRuntime().availableProcessors();
}
boolean singleThreaded = (threads == 1);
// create a thread pool with an unbounded queue to process individual files in parallel
ExecutorService exec = singleThreaded ? null : Executors.newFixedThreadPool(threads);
// an object which will be created to report the result of processing each file
class Result
{
String fileName = null;
Exception error = null;
}
// queue of future results
Queue<Future<Result>> results = new LinkedList<>();
// process the list of files
iter = safeList.listIterator();
for (int i = 0; iter.hasNext(); i++)
{
File file = iter.next();
// bypass entries that have been removed
if (file == null)
{
continue;
}
// create and configure an AstGenerator for each file to be processed
AstGenerator gen = new AstGenerator();
gen.setSilent(silent);
gen.setPreprocess(preproc);
gen.setCache(cache);
gen.setDumpLexer(lex);
gen.setDumpParser(parse);
gen.setAstPersist(persist);
Configuration config = Configuration.getInstance();
// need an effectively final variable for the list index for use within the lambda below
int index = i;
// lambda expression which processes the file on a separate thread and reports a result
Callable<Result> job = () ->
{
Result result = new Result();
File f = safeList.get(index);
if (f != null)
{
String fileName = result.fileName = f.toString();
config.withFileProfile(fileName);
try
{
boolean exists = AstManager.get().isExistingAst(AstGenerator.getAstName(fileName));
if (!silent && singleThreaded)
{
System.out.printf("%s%s\n", fileName, exists ? " (already parsed)" : "");
}
// only parse if the file has not been recursively parsed already
if (!exists)
{
// process the file, note that for HTML files the file list
// may be edited to remove the output file name that the E4GL
// preprocessor generates, if it is separately listed in
// the list (this avoids duplicate processing of that file
// since it was already processed fully to an AST when it was
// listed as an HTML file, it doesn't need to be processed
// again)
gen.processFile(fileName, safeList, index);
}
else
{
gen.markOriginal(fileName);
}
}
catch(Exception exc)
{
// some failures will raise an exception that is caught here; in other cases
// (notably within the SymbolResolver), exceptions are caught and reported,
// but not rethrown
if (singleThreaded)
{
if (!silent)
{
System.out.println("Failure in file '" + fileName + "':");
}
LOG.log(Level.SEVERE, "", exc);
if (abortOnError)
{
throw exc;
}
}
else
{
result.error = exc;
}
}
}
return result;
};
// if single-threaded, execute the lambda inline
if (singleThreaded)
{
job.call();
}
else
// submit a job for execution and queue its future result in the order submitted
{
results.add(exec.submit(job));
}
}
// all jobs are now submitted; time to process the queue of results
// ensure output to stream is in the correct order, even if the files were processed
// out of order; this can make the output seem stilted when finished jobs are stacked
// up behind an unfinished job
for (Future<Result> future : results)
{
// wait for the job to finish
Result result = future.get();
// skip any jobs that were nulled out after being submitted, due to a race condition
if (result.fileName == null)
{
continue;
}
if (!silent)
{
System.out.println(result.fileName);
}
// report any error and optionally terminate
if (result.error != null)
{
if (!silent)
{
System.out.println("Failure in file '" + result.fileName + "':");
}
LOG.log(Level.SEVERE, "", result.error);
if (abortOnError)
{
throw result.error;
}
}
}
if (!singleThreaded)
{
// all files have been processed; shut down the thread pool
exec.shutdown();
}
iter = null;
if (additionalFileList != null && additionalFileList.size() > 0)
{
files.addAdditionalFiles(additionalFileList);
additionalFileList = null;
}
if (removedConversionFiles != null)
{
files.removeFiles(removedConversionFiles);
removedConversionFiles = null;
}
// if changes have been made to the registry (by an AstGenerator)
// then we need to persist the registry (if no changes have been
// made, then this is essentially a no-op)
AstManager.get().save();
SymbolResolver.parseFinished();
}
/**
* Emit a syntax statement and optional error message to
* <code>stderr</code>, then exit the process with the specified return
* code. This method is called from {@link #main} when the user passes
* command line parameters which are not viable.
*
* @param msg
* Optional error message; may be <code>null</code>.
* @param rc
* Process return code used with <code>System.exit()</code>.
*/
private static void syntax(String msg, int rc)
{
if (msg != null)
{
LOG.log(Level.FINE, msg);
}
String[] staticText =
{
"Syntax:",
"",
" java ScanDriver [-QCLPXSANR] <directory> \"<filespec>\"",
" java ScanDriver -F[QCLPXSA] <filelist>",
"",
"Where",
"",
" options:",
" Q = run preprocessor;",
" C = cache preprocessor output;",
" L = cache lexer output;",
" P = cache parser output;",
" X = save ASTs persistently",
" F = explicit file list mode",
" S = silent mode",
" A = abort on error",
" N = no recursion in filespec mode",
" R = registry rebuild (from persistent ASTs)",
"",
" directory = a relative or absolute path (only used",
" in filespec mode)",
" filespec = file filter specification of the filenames in the",
" given directory to scan, should be enclosed in",
" double quotes if any of the wildcard characters *",
" or ? are used (only used in filespec mode)",
" filelist = arbitrary list of absolute and/or relative file",
" names to scan",
"",
"By default, options QCLPX are active. If any options are",
"explicitly specified, all non-specified options are forced off."
};
for (int i = 0; i < staticText.length; i++)
{
LOG.log(Level.FINE, staticText[i]);
}
System.exit(rc);
}
/**
* Provides a command line interface for an end user to drive the
* scanning harness.
* <p>
* Syntax:
* <pre>
* java ScanDriver [-QCLPXSANR] <directory> <"filespec">
* java ScanDriver -F[QCLPXSA] <"filelist">
* </pre>
* Where:
* <ul>
* <li> [-options] can be a hyphen followed by any combination of the
* letters 'QCLPXR' where
* <ul>
* <li> Q = run preprocessor
* <li> C = cache preprocessor output
* <li> L = cache lexer output
* <li> P = cache parser output
* <li> X = save ASTs persistently
* <li> F = interpret remaining args as an explicit file list
* <li> S = silent mode
* <li> A = abort on error",
* <li> N = no recursion in filespec mode
* <li> R = registry rebuild (from persistent ASTs)
* </ul>
* <li> filespec mode
* <ul>
* <li> <directory> is the directory to search for files
* <li> "filespec" is the file specification that will be
* used to create a list of files to process
* </ul>
* <li> explicit file list mode
* <ul>
* <li> <filelist> is arbitrary list of absolute and/or
* relative file names to scan
* </ul>
* </ul>
* <p>
* By default, options QCLPX are active. If any options are explicitly
* specified, all non-specified options are forced off.
*
* @param args
* List of command line arguments.
*/
public static void main(String[] args)
{
// option defaults
boolean preproc = true;
boolean cache = true;
boolean lex = true;
boolean parse = true;
boolean persist = true;
boolean recursion = true;
boolean silent = false;
boolean abortOnError = false;
boolean explicit = false;
int idx = 0;
// requires (directory and filespec) OR (option -F and a filelist)
if (args.length < 2)
{
syntax("Missing command line parameters.", -1);
}
// process options
if (args[0].startsWith("-"))
{
idx = 1;
String opts = args[0].toLowerCase();
if (opts.indexOf('q') != -1) preproc = true;
else preproc = false;
if (opts.indexOf('c') != -1) cache = true;
else cache = false;
if (opts.indexOf('l') != -1) lex = true;
else lex = false;
if (opts.indexOf('p') != -1) parse = true;
else parse = false;
if (opts.indexOf('x') != -1) persist = true;
else persist = false;
if (opts.indexOf('n') != -1) recursion = false;
else recursion = true;
if (opts.indexOf('s') != -1) silent = true;
else silent = false;
if (opts.indexOf('a') != -1) abortOnError = true;
else abortOnError = false;
if (opts.indexOf('f') != -1) explicit = true;
else explicit = false;
}
FileList files = null;
// at this point idx points to the next arg after the options
if (explicit)
{
int len = args.length;
String[] filelist = new String[len - idx];
for (int j = idx; j < len; j++)
{
filelist[j - 1] = args[j];
}
files = new ExplicitFileList(filelist);
}
else
{
// options with filespec mode requires 3 args in total
if (idx == 1 && args.length < 3)
{
syntax("Missing command line parameters.", -2);
}
files = new FileSpecList(new File(args[idx]),
args[idx+1],
recursion);
}
// drive the scanning
try
{
scan(files, preproc, cache, lex, parse, persist, silent, abortOnError, 0);
}
catch (Exception exc)
{
// non-zero return to indicate error
System.exit(1);
}
}
}