# TransformDriver.java

    /*
    ** Module   : TransformDriver.java
    ** Abstract : common code for transformation of a file or set of files
    **
    ** Copyright (c) 2006-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --------------------------------------Description---------------------------------------
    ** 001 GES 20181022 Extracted common code from ConversionDriver.
    ** 002 ECF 20190218 Added blacklist (filespec + ignore list) mode and dry-run mode (only list source files
    **                  to be processed, but do nothing more).
    ** 003 CA  20190703 Legacy 4GL classes are always processed in BFS order by the conversion driver. They are
    **                  added at the beginning of the list, with all other files in normal order, following them.
    ** 004 ECF 20190528 Ignore comments in ignore list (demarcated by a leading '#' character).
    ** 005 GES 20191011 Fix for processing Progress.Lang.Object.
    ** 006 CA  20200412 Added incremental conversion support.
    **     RFB 20200430 Fixed minor typo. Added some extra text to make added stand out.
    ** 007 CA  20200930 Force ANTLR to use Class.forName instead of loadClass.
    **     CA  20210213 Normalize the files read from the explicit or implicit file list, so that their names will
    **                  use the OS separator.  The explicit or implicit file list will always use the Linux-style
    **                  '/' separator.
    **     CA  20210423 'tmpTabNames' has as value an AST which may be from a different file.  For incremental
    **                  conversion, if we include a file which affects values in this map, then all affected
    **                  keys must be considered as dependencies and their files reconverted, too.
    **     GES 20210707 Removed some dead code and moved to use of some common logic in the FileList class.
    **     OM  20210923 Added -profile command line option for specifying the configuration profile.
    **     CA  20211012 Fixed generated WSDL and reworked SOAP support to rely on namespace, when resolving an
    **                  operation.
    **     OM  20220325 Dynamic resolution of .dict, .schema and .p2o files as opposed to hardcoded path.
    **     OM  20220330 Moved conversion artifacts to ${cvtpath} folder.
    **     GES 20220320 Moved some of the inline file list processing back into new FileListFactory workers.
    **     CA  20220412 Added file-set support (either -Z command-line option or p2j.cfg.xml configuration).
    **     CA  20220419 The options are really optional, if no option is set, then only 'mode' is allowed and the
    **                  p2j.cfg.xml is checked for a file-set (assuming 'Z' mode).
    **     TJD 20220504 Java 11 compatibility minor changes
    **     CA  20220501 Each profile has the same structure as the main 'global' config.  Allow multiple profiles
    **                  to be ran at once, with the conversion switching the state between profiles, when a
    **                  resource (like a file or namespace) is being processed.  Only front phase is supported at
    **                  this time.
    **     CA  20220516 Made JobDefinition protected.
    **     CA  20220524 For incremental conversion, normalize the filenames, as this is how they are kept in the
    **                  conversion database.
    **     GES 20210520 Sort interfaces into the BFS class hierachy so that all overridden methods will be
    **                  converted in the correct order (the parent version first).
    ** 008 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    ** 009 OM  20230115 Replaced absolutePath(), relativePath(), upPath() and downPath() with faster
    **                  versions, based on node types rather on string paths.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    /*
     TODOS:
     - separate modes for various cleanup (deletion of different sets of output)
     - more granular debug options (step specific instead of global)
    */

    package com.goldencode.p2j.convert;

    import java.io.*;
    import java.util.*;
    import java.util.logging.*;
    import java.text.*;

    import com.goldencode.ast.*;
    import com.goldencode.util.*;
    import com.goldencode.p2j.cfg.*;
    import com.goldencode.p2j.convert.db.*;
    import com.goldencode.p2j.uast.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.p2j.pattern.*;
    import com.goldencode.p2j.schema.*;

    /**
     * Provides a harness to drive the end-to-end transformation process for a user-defined list of files.
     */
    public abstract class TransformDriver
    {
       /** No message output - quiet mode. */
       public static final int MSG_NONE = 0;

       /** Output status messages only. */
       public static final int MSG_STATUS = 1;

       /** Output debug and status messages.  */
       public static final int MSG_DEBUG = 2;

       /** Full debug trace output. */
       public static final int MSG_TRACE = 3;

       /** Spacer for section headers and output. */
       protected static final String spacer = StringHelper.repeatChar('-', 78);

       /** Match list for database name conversion. */
       protected static final String datanames = Configuration.getParameter("datanames");

       /** Chars that must be escaped to eliminate regular expression meaning. */
       protected static final char[] DANGER_LIST = { '{', '}', '[', ']', '(', ')' };

       /** Form in which to obtain the list of files. */
       protected static enum ListType { CMD_LINE, WHITELIST, BLACKLIST, FILESPEC, FILESET };

       /** Logger */
       private static final ConversionStatus LOG = ConversionStatus.get(TransformDriver.class);

       static
       {
          try
          {
             String name = Configuration.getParameter("registry");
             AstManager.initialize(() -> new XmlFilePlugin(name, Configuration.isImportMode()));
          }
          catch (AstException ae)
          {
             LOG.log(Level.SEVERE, "Conversion initialization failed!", ae);
          }
       }

       /** The list of Progress source files (command line input). */
       protected FileList source = null;

       /** The list of Progress AST (abstract syntax tree) files. */
       protected FileList asts = null;

       /** The list of database dictionary files. */
       protected FileList dicts = null;

       /** The list of Progress source file specific dictionary files. */
       protected FileList sdicts = null;

       /** The debug level to honor throughout the process. */
       protected int debug = MSG_STATUS;

       /** The details of which features this job run must implement. */
       protected RunMode mode = null;

       /** Full details of this run. */
       protected JobDefinition job = null;

       /** Base class for all OO classes. */
       private static final String ROOT_CLASS = "Progress.Lang.Object".toLowerCase();

       /**
        * Creates an instance with a specific configuration.
        *
        * @param    job
        *           The details of the job to execute.
        */
       public TransformDriver(JobDefinition job)
       {
          this.source = job.files;
          this.mode   = job.mode;
          this.job    = job;

          if (job.debug < MSG_NONE || job.debug > MSG_TRACE)
          {
             throw new IllegalArgumentException("Invalid debug level: " + job.debug);
          }
          else
          {
             this.debug = job.debug;
          }
       }

       /**
        * Drive the front end of the conversion process using the file list
        * of Progress source files that is already defined in this instance.
        * <p>
        * Once the scanning of the source files is complete, this processing will
        * modify the files list to create one that exposes AST files instead of
        * Progress source files.
        * <p>
        * This runs the following:
        * <ul>
        *    <li> schema loader (optional)
        *    <li> preprocessor (optional)
        *    <li> lexer
        *    <li> parser
        *    <li> AST persister
        *    <li> schema fixups for the dictionary and for source-specific schema
        *         files (optional)
        *    <li> call graph generator (optional)
        *    <li> saves off a set results for enabling a future restore to this
        *         point
        * </ul>
        *
        * @throws   ConfigurationException   If there is problem reading/parsing the configuration.
        * @throws   AstException             If there is an problem processing ASTs.
        * @throws   SchemaException          If there is a problem with the schema.
        * @throws   IOException              If there is a problem with I/O.
        * @throws   Exception                For any other problem.
        */
       protected void front()
       throws ConfigurationException,
              AstException,
              SchemaException,
              IOException,
              Exception
       {
          if (mode.schema)
          {
             // run schema loader to generate new dictionary files
             runSchemaLoader();
          }

          // optionally preprocess, lex, parse and save the resulting ASTs as
          // persisted XML files with the name sourcename.[pPwW].ast
          runScanDriver(mode.preproc, mode.schema, job.abortOnError, job.threads);

          // ensure that downstream processing has the proper list of files
          convertSourceNamesToAstNames();

          processTrees("Post-Parse Fixups",
                       "fixups/post_parse_fixups",
                       null,
                       false,
                       asts,
                       debug);

          processTrees("Early Annotations",
                       "annotations/early_annotations",
                       null,
                       false,
                       asts,
                       debug);

          processTrees("Gap Analysis Marking",
                       "gaps/gap_analysis_marking",
                       null,
                       false,
                       asts,
                       debug);

          if (mode.schema)
          {
             // clean up previous run - build schema list and delete those files
             FileList delDicts = createDatabaseList(".schema", false);
             deleteFiles(delDicts.listFilenames());

             // build dictionary list
             dicts = createDatabaseList(".dict", omitSchemaMetadata());

             // fixup dictionary files
             processTrees("Schema Fixups (data dictionary)",
                          "schema/fixups",
                          datanames,
                          true,
                          dicts,
                          debug);

             // build schema list
             sdicts = convertSourceNamesToDictNames(".dict");

             // fixup source file schema files (if any exist for these specific
             // source files)
             if (sdicts.size() > 0)
             {
                processTrees("Schema Fixups (Progress source file schemas)",
                             "schema/fixups",
                             datanames,
                             true,
                             sdicts,
                             debug);
             }
          }

          if (mode.callgraph)
          {
             runCallGraphGenerator();
          }
       }

       /**
        * Runs the <code>SchemaLoader</code> class to convert <code>.df</code>
        * files into an AST form which can be read by the
        * <code>SchemaDictionary</code>.  Files left by previous runs, if any,
        * are removed first.
        *
        * @throws   ConfigurationException   If there is problem reading/parsing the configuration.
        * @throws   SchemaException          If there is a problem with the schema.
        * @throws   IOException              If there is a problem with I/O.
        */
       protected void runSchemaLoader()
       throws ConfigurationException,
              SchemaException,
              IOException
       {
          printHeader("SchemaLoader", debug);

          // clean up previous run - build dictionary list and delete those files
          FileList delDicts = createDatabaseList(".dict", false);
          deleteFiles(delDicts.listFilenames());

          SchemaLoader loader = new SchemaLoader();

          if (debug == MSG_NONE)
          {
             loader.setQuietMode(true);
          }

          loader.importAll();
       }

       /**
        * Runs the <code>ScanDriver</code> class to convert Progress source
        * files into an AST form.  This corresponds to preprocessing, lexing,
        * parsing and AST persistence.
        *
        * @param    preproc
        *           <code>true</code> if the <code>Preprocessor</code> should
        *           should be forced to run.  <code>false</code> if previously
        *           cached preprocessor results should be honored.
        * @param    schema
        *           <code>true</code> if the <code>SchemaLoader</code> and
        *           <code>schema/fixups.rules</code> should be run.  Note this
        *           is only to assist with cleaning files as the schema
        *           processing itself is not handled here.
        * @param    abortOnError
        *           <code>true</code> to end the scanning run if an error is encountered,
        *           else <code>false</code>.
        * @param    threads
        *           Number of threads to use for the scan. A non-positive value will enable
        *           the default behavior of a work-stealing thread pool with one thread per
        *           per available CPU. A positive value will create a fixed size thread pool.
        *
        * @throws   Exception                If there is a problem.
        */
       protected void runScanDriver(boolean preproc,
                                    boolean schema,
                                    boolean abortOnError,
                                    int threads)
       throws Exception
       {
          printHeader("Scanning Progress Source (" +
                      (preproc ? "preprocessor, " : "") +
                      "lexer, parser, persist ASTs)",
                      debug);

          long start = System.currentTimeMillis();

          cleanScanResults(preproc, schema);

          ScanDriver.scan(source, (debug == MSG_NONE), abortOnError, threads);

          printElapsed(System.currentTimeMillis() - start);
       }

       /**
        * Remove some or all scan driver output files associated with the
        * current Progress source file list AND in the list of fakeout files
        * that represent the built-in 4GL classes/interfaces or .NET classes
        * that can be accessed.  All output except preprocessor and schema output
        * is unconditionally deleted.
        *
        * @param    preproc
        *           <code>true</code> means that cached preprocessor files should
        *           be deleted.
        * @param    schema
        *           <code>true</code> means that schema related generated files
        *           should be deleted.
        *
        * @throws   IOException              If there is a problem with I/O.
        */
       protected void cleanScanResults(boolean preproc, boolean schema)
       throws IOException
       {
          ArrayList<String> exts = new ArrayList<>();

          exts.add(".lexer");
          exts.add(".parser");
          exts.add(".ast");
          exts.add(".jast");

          if (preproc)
          {
             exts.add(".cache");
             exts.add(".pphints");
          }

          if (schema)
          {
             exts.add(".dict");
             exts.add(".schema");
             exts.add(".schema.original");
             exts.add(".p2o");
          }

          // we never delete ".schema.hints" or ".hints" as those are both
          // hand-coded configuration files

          String[] extslist = exts.toArray(new String[0]);

          // do the work
          zapFiles(source.listFilenames(), extslist);
          if (!job.incremental)
          {
             zapFiles(SymbolResolver.listFakeoutFiles(), extslist);
          }
       }

       /**
        * Concatenate all extensions with all the files and if any resulting
        * names exist, delete the matching file from the file system.
        *
        * @param    files
        *           The list of file base names.
        * @param    extslist
        *           The list of extensions to zap.
        */
       protected void zapFiles(String[] files, String[] extslist)
       {
          for (int i = 0; i < files.length; i++)
          {
             for (int j = 0; j < extslist.length; j++)
             {
                File check = new File(files[i] + extslist[j]);

                if (check.exists() && check.isFile())
                {
                   // delete the file
                   check.delete();
                }
             }
          }
       }

       /**
        * Process each input element and escape characters that would be
        * interpreted as part of a regular expression to convert them all into
        * a literal match.  At this time, only a limited subset of characters
        * is escaped (curly braces, square brackets and parenthesis). This allows
        * valid file name characters to be passed literally in a larger file
        * specification that will be interpreted as a regular expression.
        *
        * @param    list
        *           List of literal strings that may need to be escaped.
        *
        * @return   Safe version of the list.
        */
       protected String[] escapeRegex(String[] list)
       {
          String[] result = new String[list.length];

          for (int i = 0; i < list.length; i++)
          {
             StringBuilder sb = new StringBuilder();

             int len = list[i].length();

             for (int j = 0; j < len; j++)
             {
                char next = list[i].charAt(j);

                for (int x = 0; x < DANGER_LIST.length; x++)
                {
                   if (DANGER_LIST[x] == next)
                   {
                      sb.append('\\');
                      break;
                   }
                }

                sb.append(next);
             }

             result[i] = sb.toString();
          }

          return result;
       }

       /**
        * Runs the <code>CallGraphGenerator</code> using the project's defined
        * root nodes list to create a call graph for each root node.
        *
        * @throws   ConfigurationException   If there is problem reading/parsing the configuration.
        * @throws   AstException             If there is an problem processing ASTs.
        * @throws   IOException              If there is a problem with I/O.
        */
       protected void runCallGraphGenerator()
       throws ConfigurationException,
              IOException,
              AstException
       {
          printHeader("Generating Call Graphs", debug);

          try
          {
             CallGraphGenerator.generateGraphs(null, debug);
          }
          finally
          {
             CallGraphHelper.shutdownGraph();
          }
       }

       /**
        * Run the pattern engine using the given rules filename as the program
        * and the list of files are the ASTs (abstract syntax trees) to process.
        *
        * @param    section
        *           The section name (for the header);
        * @param    rules
        *           The pattern engine program (known as a rules file) to run.
        * @param    matches
        *           The match list name to use or <code>null</code> to use the
        *           project default.
        * @param    db
        *           Load default database name mappings.
        * @param    files
        *           The list of ASTs to process.
        * @param    debug
        *           The debug level to use with the pattern engine.
        *
        * @return   The list of 4GL program names generated by this engine run, reported by
        *           {@link PatternEngine#getNewProgramFiles()}.
        *
        * @throws   ConfigurationException   If there is problem reading/parsing the configuration.
        * @throws   AstException             If there is an problem processing ASTs.
        */
       protected static List<String> processTrees(String   section,
                                                  String   rules,
                                                  String   matches,
                                                  boolean  db,
                                                  FileList files,
                                                  int      debug)
       throws ConfigurationException,
              AstException
       {
          printHeader(section, debug);

          // load/reload a known match list for name conversion
          NameConverter.load(matches, db);

          // create a new pattern engine, configure it and run
          PatternEngine engine = new PatternEngine();
          engine.addAstSpec(files);
          PatternEngine.setDebugLevel(debug);
          engine.run(rules);

          return engine.getNewProgramFiles();
       }

       /**
        * Run the pattern engine using the given rules filename as the program
        * and the list of files are the ASTs (abstract syntax trees) to process.
        * <p>
        * Also, set the variable initializers using the given map.
        *
        * @param   section
        *          The section name (for the header);
        * @param   rules
        *          The pattern engine program (known as a rules file) to run.
        * @param   matches
        *          The match list name to use or <code>null</code> to use the
        *          project default.
        * @param   db
        *          Load default database name mappings.
        * @param   files
        *          The list of ASTs to process.
        * @param   debug
        *          The debug level to use with the pattern engine.
        * @param   vars
        *          The variable initializers.
        *
        * @throws  ConfigurationException
        *          If there is problem reading/parsing the configuration.
        * @throws  AstException
        *          If there is a problem processing ASTs.
        */
       protected void processTrees(String   section,
                                   String   rules,
                                   String   matches,
                                   boolean  db,
                                   FileList files,
                                   int      debug,
                                   Map      vars)
       throws ConfigurationException,
              AstException
       {
          printHeader(section, debug);

          // load/reload a known match list for name conversion
          NameConverter.load(matches, db);

          // create a new pattern engine, configure it and run
          PatternEngine engine = new PatternEngine();
          engine.addAstSpec(files);
          PatternEngine.setDebugLevel(debug);
          engine.setVariableInitializers(vars);
          engine.run(rules);
       }

       /**
        * Delete the file associated with each name in the list.
        *
        * @param   list
        *          List of explicit filenames to delete.
        *
        * @throws  IOException
        *          If there is a problem with I/O.
        */
       protected void deleteFiles(String[] list)
       throws IOException
       {
          for (int i = 0; i < list.length; i++)
          {
             File next = new File(list[i]);
             next.delete();
          }
       }

       /**
        * For each element in the spec list, generate a list of matching files
        * (each spec may contain traditional file system wildcard characters)
        * and delete each of these files.  The given <code>ExcludeCheck</code>
        * instance is used to remove results from the list on a callback basis.
        *
        * @param   speclist
        *          List of file specifications.
        * @param   check
        *          If <code>null</code>, no callback occurs.  Otherwise, the
        *          <code>exclude</code> method will be called on the passed instance.
        *
        * @throws  IOException
        *          If there is a problem with I/O.
        */
       protected void deleteFileSpecs(String[] speclist, ExcludeCheck check)
       throws IOException
       {
          for (int i = 0; i < speclist.length; i++)
          {
             int    idx  = speclist[i].lastIndexOf(File.separator);
             String dir  = (idx < 0) ? "." : speclist[i].substring(0, idx);
             String spec = speclist[i].substring(idx + 1);

             // create a list of actual files matching this spec
             FileList list = new FileSpecList(new File(dir), spec, false);
             File[]   fset = list.list();

             for (int j = 0; j < fset.length; j++)
             {
                // prune out the preprocessor cache files
                if (check != null && check.exclude(fset[j].getCanonicalPath()))
                {
                   continue;
                }

                // delete the file
                fset[j].delete();
             }
          }
       }

       /**
        * Print a section header and optionally spacers, depending on debug
        * level (if level &gt; <code>MSG_NONE</code> then spacers are printed).
        *
        * @param    section
        *           The section name.
        * @param    debug
        *           The debug level.
        */
       protected static void printHeader(String section, int debug)
       {
          if (debug > MSG_NONE)
          {
             System.out.println();
             System.out.println(spacer);
          }

          System.out.println(section);

          if (debug > MSG_NONE)
          {
             System.out.println(spacer);
             System.out.println();
          }
       }

       /**
        * Using the source name list as input, this method creates duplicate
        * lists with the proper <code>.ast</code> or <code>.jast</code> file
        * suffixes on each entry. The results are saved into the Progress AST
        * name list.
        */
       protected void convertSourceNamesToAstNames()
       {
          List<String> astFiles = new ArrayList<>();

          astFiles.addAll(Arrays.asList(source.withSuffix(".ast")));

          asts = new ExplicitFileList(astFiles.toArray(new String[0]));
          Set<String> files = new LinkedHashSet<>();

          // preserve the order as listed by FileList
          files.addAll(Arrays.asList(asts.listFilenames()));

          // get the legacy class hierarchy and create a BFS walk.  these will be processed further
          // in this order
          List<String> classes = sortLegacyClasses(files);

          List<String> sorted = new ArrayList<>(classes);
          files.removeAll(classes);
          sorted.addAll(files);

          asts = new ExplicitFileList(sorted.toArray(new String[0]));
          ((ExplicitFileList) asts).setSort(false);
       }

       /**
        * Scans the configuration file and create a list of artefacts containing a specific file for each declared
        * namespace from the default profile, if such file exists.
        * <br>
        * While the extension is common for all databases, the location where the files are searched is specific
        * to each database (namespace) and is computed using the {@code xmlFile} attribute of the
        * {@code namespace} because all these artefacts are generated in same location.
        * <br>
        * By default, if no {@code xmlFile} attribute is specified for a namespace in the configuration, the
        * project's {@code data/namespace} is assumed.
        *
        * @param   extension
        *          The file extension to search for (usually {@code .dict}, {@code .schema}, or {@code .p2o}).
        * @param   omitMeta
        *          {@code true} to omit the metadata filename from the resulting list; {@code false} to retain it.
        *
        * @return  The list of files found.
        *
        * @throws  ConfigurationException
        *          If there is problem reading/parsing the configuration.
        */
       protected FileList createDatabaseList(String extension, boolean omitMeta)
       throws ConfigurationException
       {
          Configuration cfg = Configuration.getInstance();
          SchemaConfig config = Configuration.getSchemaConfig();
          String metaName = config.getMetadata().getName(); // default is "standard"

          List<String> toReturn = new ArrayList<>();
          Iterator<String> it = config.databases(cfg.isMultiProfile());
          while (it.hasNext())
          {
             String schema = it.next();

             if (cfg.isMultiProfile())
             {
                cfg.withDbProfile(schema, (profile) ->
                {
                   String profileMetaName = config.getMetadata().getName();

                   // omit meta if so required
                   if (omitMeta && schema.equals(profileMetaName))
                   {
                      return;
                   }

                   String fileName = config.getSchemaFileName(schema, extension);
                   if (new File(fileName).exists() && !toReturn.contains(fileName))
                   {
                      // collect ONLY existing files
                      toReturn.add(fileName);
                   }
                });
                continue;
             }

             // omit meta if so required
             if (omitMeta && schema.equals(metaName))
             {
                continue;
             }

             String fileName = config.getSchemaFileName(schema, extension);
             if (new File(fileName).exists() && !toReturn.contains(fileName))
             {
                // collect ONLY existing files
                toReturn.add(fileName);
             }
          }

          return new ExplicitFileList(toReturn.toArray(new String[toReturn.size()]));
       }

       /**
        * Determine whether we should omit the metadata schema when processing and persisting schema
        * information.
        *
        * @return  <code>true</code> if no metadata tables are specified by the project configuration,
        *          else <code>false</code>.
        */
       protected boolean omitSchemaMetadata()
       {
          return Configuration.getSchemaConfig().getMetadata().getTables().isEmpty();
       }

       /**
        * Using the source name list as input, this method creates a duplicate
        * list with the given file suffix on each entry.
        *
        * @param    suffix
        *           The file extension to search for (usually <code>.dict</code>
        *           or <code>.schema</code>).
        *
        * @return   The list of files found.
        */
       protected FileList convertSourceNamesToDictNames(String suffix)
       {
          return new ExplicitFileList(source.withSuffix(suffix));
       }

       /**
        * Replace the filename extension with the given extension and return the new string. If the
        * given filename already has the given extension, simply return the existing filename.
        *
        * @param   filename
        *          File name;  may not be <code>null</code>.
        * @param   append
        *          Extension to append after the filename's existing extension is removed. Must
        *          begin with the dot (<code>.</code>) character.
        *
        * @return  Updated filename.
        */
       protected String replaceFilenameExtension(String filename, String append)
       {
          if (filename.endsWith(append))
          {
             return filename;
          }

          StringBuilder buf = new StringBuilder();
          int pos = filename.lastIndexOf(".");
          if (pos < 0)
          {
             buf.append(filename);
          }
          else
          {
             buf.append(filename, 0, pos);
          }

          buf.append(append);

          return buf.toString();
       }

       /**
        * Handle the execution of the front, middle and back of the transformation process
        * as defined in the job details.
        *
        * @param    title
        *           Title for this driver's job.
        */
       protected void executeJob(String title)
       {
          // set a global property to for Antlr to use Class.forName
          System.setProperty("ANTLR_USE_DIRECT_CLASS_LOADING", "true");

          // set the configuration profile
          Configuration.setDefaultProfiles(job.cfgProfiles);

          // capture the start time
          long jobStartTime = System.currentTimeMillis();

          try
          {
             // print the main header
             printHeader(title, MSG_STATUS);

             if (job.incremental)
             {
                printHeader("Incremental Conversion - checking file signature", MSG_STATUS);

                ConversionData.connect();

                // filter the files and keep only the following:
                // - newly added files
                // - changed files
                String[] files = job.files.listFilenames();
                for (int i = 0; i < files.length; i++)
                {
                   files[i] = Configuration.normalizeFilename(files[i]);
                }
                Set<String> filteredFiles = new LinkedHashSet<>();
                Set<String> dependencies = new LinkedHashSet<>();
                dependencies.addAll(Arrays.asList(files));

                Set<String> depFiles = new HashSet<>();
                boolean force = false;

                do
                {
                   depFiles.clear();
                   for (String file : dependencies)
                   {
                      if (force || ConversionData.mustConvertFile(file))
                      {
                         System.out.println(file + " - added for conversion" + (force ? " (dependency)" : ""));
                         filteredFiles.add(file);

                         depFiles.addAll(ConversionData.clean(file));
                      }
                      else
                      {
                         System.out.println(file + " - unchanged");
                      }
                   }

                   depFiles.removeAll(filteredFiles);

                   dependencies.clear();
                   dependencies.addAll(depFiles);

                   filteredFiles.addAll(depFiles);

                   force = true;
                }
                while (!depFiles.isEmpty());

                ConversionData.disconnect();

                if (filteredFiles.isEmpty())
                {
                   System.out.println("INCREMENTAL CONVERSION: no changed files found");
                   return;
                }

                job.files = new ExplicitFileList(filteredFiles.toArray(new String[0]),
                                                 job.files.isCaseSensitive());
                source = job.files;
             }
             else
             {
                ConversionData.cleanAll();
             }

             // front-end
             if (mode.front)
             {
                front();
             }

             // middle
             if (mode.middle)
             {
                middle();
             }

             // back-end
             if (mode.back)
             {
                back();
             }
          }

          catch (Exception exc)
          {
             LOG.log(Level.SEVERE, "ERROR:", exc);
             System.exit(1);
          }

          finally
          {
             ConversionData.disconnect();

             // print out total time
             printElapsed(System.currentTimeMillis() - jobStartTime);
          }
       }

       /**
        * Drive the middle of the transformation process using the file list of Progress AST files
        * that is already defined in this instance.
        *
        * @throws   ConfigurationException   If there is problem reading/parsing the configuration.
        * @throws   AstException             If there is an problem processing ASTs.
        * @throws   SchemaException          If there is a problem with the schema.
        * @throws   IOException              If there is a problem with I/O.
        */
       protected abstract void middle()
       throws ConfigurationException,
              AstException,
              SchemaException,
              IOException;

       /**
        * Drive the back end of the transformation process using the file list of Progress AST files
        * that is already defined in this instance.
        *
        * @throws   ConfigurationException   If there is problem reading/parsing the configuration.
        * @throws   AstException             If there is an problem processing ASTs.
        * @throws   SchemaException          If there is a problem with the schema.
        * @throws   IOException              If there is a problem with I/O.
        */
       protected abstract void back()
       throws ConfigurationException,
              AstException,
              SchemaException,
              IOException;

       /**
        * Enables delegation of list processing to another method while providing
        * a callback to determine if a given list entry needs to be excluded from
        * processing.
        */
       protected abstract class ExcludeCheck
       {
          /**
           * Determines if a given name should be excluded from list processing.
           *
           * @param    name
           *           The list element to check.
           *
           * @return   <code>true</code> if the element should be excluded from
           *           list processing.
           */
          abstract boolean exclude(String name);
       }

       /**
        * Helper to only exclude the generated preprocessor files (cache and
        * preprocessor hints) conditionally and to always exclude the manually
        * coded hints files.
        */
       protected class ScanCheck
       extends ExcludeCheck
       {
          /** <code>true</code> if preprocessor files should be deleted. */
          protected boolean preproc = false;

          /** <code>true</code> if schema files should be deleted. */
          protected boolean schema = false;

          /**
           * Construct and instance and store the preprocessor delete setting.
           *
           * @param    preproc
           *           <code>true</code> if preprocessor files should be deleted.
           * @param    schema
           *           <code>true</code> if schema files should be deleted.
           */
          ScanCheck(boolean preproc, boolean schema)
          {
             this.preproc = preproc;
             this.schema  = schema;
          }

          /**
           * Determines if a given name should be excluded from list processing.
           *
           * @param    name
           *           The list element to check.
           *
           * @return   <code>true</code> if the element should be excluded from
           *           list processing.
           */
          boolean exclude(String name)
          {
             return (!preproc && name.endsWith(".cache"))           ||
                    (!preproc && name.endsWith(".pphints"))         ||
                    (!schema  && name.endsWith(".dict"))            ||
                    (!schema  && name.endsWith(".schema"))          ||
                    (!schema  && name.endsWith(".schema.original")) ||
                    (!schema  && name.endsWith(".p2o"))             ||
                    name.endsWith(".schema.hints")                  ||
                    name.endsWith(".hints");
          }
       }

       /**
        * Interpret the mode command line argument and convert this into the
        * effective run mode object.  Multiple "+" delimited mode values can
        * be included in the same argument.  In this case, the result is the
        * logical OR of all the activated flags in each specified mode.
        *
        * @param    key
        *           The mode command line argument.
        * @param    modes
        *           Valid run modes.
        * @param    help
        *           Help text for syntax display.
        *
        * @return   The run mode to use for controlling program flow.
        */
       protected static RunMode interpretRunMode(String               key,
                                                 Map<String, RunMode> modes,
                                                 String[]             help)
       {
          String[] cand   = key.split("\\+");
          RunMode  result = null;

          if (key.length() == 0 || cand.length == 0)
          {
             syntax(help, "Missing mode parameter.", -4);
          }

          for (int i = 0; i < cand.length; i++)
          {
             RunMode next = modes.get(cand[i]);

             if (next == null)
             {
                syntax(help, "Unrecognized mode '" + key + "'.", -5);
             }

             if (result == null)
             {
                // our first match sets the starting result
                result = next;
             }
             else
             {
                // logically OR these together so that the result is the
                // combination of all true flags
                result = result.merge(next);
             }
          }

          return result;
       }

       /**
        * Print out a simple elapsed time counter.
        *
        * @param    elapsed
        *           Number of milliseconds of time elapsed.
        */
       protected static void printElapsed(long elapsed)
       {
          Date date = new Date(elapsed - TimeZone.getDefault().getRawOffset());
          SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");

          System.out.println();
          System.out.println(spacer);
          System.out.println("Elapsed job time:  " + format.format(date));
          System.out.println();
       }

       /**
        * Emit a syntax statement and optional error message to <code>stderr</code>, then exit the
        * process with the specified return code. This method is called when the user passes command
        * line parameters which are not viable.
        *
        * @param    helpText
        *           Help text for syntax display.
        * @param    msg
        *           Optional error message; may be <code>null</code>.
        * @param    rc
        *           Process return code used with <code>System.exit()</code>.
        */
       protected static void syntax(String[] helpText, String msg, int rc)
       {
          if (msg != null)
          {
             LOG.log(Level.SEVERE, msg);
          }

          if (helpText != null)
          {
             LOG.log(Level.WARNING, String.join(System.lineSeparator(), helpText));
          }

          System.exit(rc);
       }

       /**
        * Container for the details of a driver run.
        */
       protected static class JobDefinition
       {
          /** Flag indicating if the incremental conversion mode is on. */
          public final boolean incremental;

          /** 4GL source files to process. */
          public /*final*/ FileList files;

          /** Debug level. */
          public final int debug;

          /** Defines the features of this job. */
          public final RunMode mode;

          /** Abort on error flag. */
          public final boolean abortOnError;

          /**
           * Number of threads to use for the scan. A non-positive value will enable the default behavior of a
           * work-stealing thread pool with one thread per available CPU. A positive value will create a fixed
           * size thread pool.
           */
          public final int threads;

          /** User-defined rule-set. */
          public final String transform;

          /** The optional configuration profile(s). */
          public final List<String> cfgProfiles;

          /**
           * Defines a new job.
           *
           * @param   run
           *          Defines the features of this job.
           * @param   transform
           *          User-defined rule-set.
           * @param   files
           *          The 4GL source files to process.
           * @param   cfgProfiles
           *          The optional configuration profiles.
           * @param   threads
           *          Number of threads to use for the scan.
           * @param   abortOnError
           *          Abort on error flag.
           * @param   incremental
           *          Flag indicating if the incremental conversion mode is on.
           * @param   debug
           *          The debug level.
           */
          protected JobDefinition(RunMode run,
                                  String transform,
                                  FileList files,
                                  List<String> cfgProfiles,
                                  int threads,
                                  boolean abortOnError,
                                  boolean incremental,
                                  int debug)
          {
             this.mode = run;
             this.transform = transform;
             this.files = files;
             this.cfgProfiles = cfgProfiles == null || cfgProfiles.isEmpty() ? null : new ArrayList<>(cfgProfiles);
             this.threads = threads;
             this.abortOnError = abortOnError;
             this.incremental = incremental;
             this.debug = debug;
          }
       }

       /**
        * Process the command line interface for the driver.
        * <p>
        * The program will return a non-zero code if there is a fatal error during the transformation processing.
        * Each child class implements its own command line syntax, but the options, modes and 4GL source file
        * lists are handled in a common way.  The mode names are driver- specific, however the various front end
        * modes are common since all drivers need to be able to create ASTs.
        * <p>
        * Syntax:
        * <pre>{@code java -DP2J_HOME=<home> <TransformDriverChildClass> [-options]* <mode> [user_defined_rule_set] <file-specs>}</pre>
        * Where:
        * <ul>
        *   <li>Options (F, S, and X are mutually exclusive. If none is specified <file-specs> is a list of files
        *       added to command line):
        *       <ul>
        *          <li>Dn = set debug level 'n' (must be a numeric digit between {@code MSG_NONE} (0) and
        *              {@code MSG_TRACE} (3) inclusive
        *          <li>I = incremental conversion; only changed/new programs will be parsed and converted.
        *          <li>F = {@code <file-specs>} is a {@code <filename>} containing a custom FILE-LIST, instead of
        *              using the explicit command line file list
        *          <li>S = {@code <file-specs>} is composed of a root {@code <directory>} and a {@code
        *              "<file-filter>"}, to select SPECIFIC files instead of an explicit command line file list
        *          <li>X = {@code <file-specs>} is similar to S but has an additional custom EXCLUDE list file
        *              {@code "<filename>"} to be ignored
        *          <li>N = NO recursion in {@code <file-filter>} (S/X options)
        *          <li>Z = {@code <file-set>} is a {@code <filename>} containing the definition file list and
        *              Include plus eXclude operations to be applied on it
        *          <li>A = ABORT scan part of front end on an error
        *          <li>Tn = specify number of THREADS 'n' to use for front end file scan ('n' must be {@code >=}
        *              0; defaults to 1; set to 0 to use 1 thread per available CPU)
        *              WARNING: this mode is not compatible if there are legacy .cls files
        *          <li>L = LIST files to be processed only; do not perform any further task
        *          <li>Pname = {@code <name>} of the configuration PROFILE follows the option.  Can appear
        *              multiple times, to specify which profiles are included in this run.
        *       </ul>
        *   <li>{@code mode} (one or more of the following values, use '+' to delimit if there are two or more
        *       values; don't insert spaces!)
        *   <li>The optional {@code user_defined_rule_set} specifies the user-defined rule-set filename to execute
        *        (only some modes support this, see sub-class documentation for details)
        *   <li>explicit command line list (default approach if none of S, P, or F are present)
        *       <ul>
        *          <li>{@code <filelist>} is arbitrary list of absolute and/or relative file names to scan, this
        *              list is hard coded in the command line itself
        *          <li>any number of files may be listed on the command line, subject to command shell limits
        *       </ul>
        *   <li>custom file list (FILE approach)
        *       <ul>
        *          <li>{@code <filename>} is a single absolute or relative filename of a text file that contains a
        *              custom list of absolute and/or relative file names of files to scan
        *          <li>in other words, the file list will be read from a file instead of being hard coded on the
        *              command line
        *          <li>there must be one filename per line in the file
        *          <li>there is no limit of the number of files in the list
        *       </ul>
        *   <li>file-spec mode (expressly SPECIFIED files approach)
        *       <ul>
        *          <li>{@code <directory>} is the root directory to search for files
        *          <li>{@code "filespec"} is the file specification that will be used to create a list of files to
        *              process, should be enclosed in double quotes if any of the wildcard characters '*' or '?'
        *              are used
        *       </ul>
        *       If no files are specified, then the {@code <file-set>} configuration from {@code p2j.cfg.xml} will
        *       be used (the Z mode is used implicitly).  If there is no file-set, then the program stops.
        *   <li>custom ignore list (file-spec mode with additional EXCLUDE list)
        *       <ul>
        *          <li>{@code <directory>} is the root directory to search for files
        *          <li>{@code "filespec"}" is the file specification that will be used to create a list of files
        *              to process, should be enclosed in double quotes if any of the wildcard characters '*' or
        *              '?' are used
        *          <li>{@code <filename>} is a single absolute or relative filename of a text file that contains a
        *              custom list of specifications of file names to ignore
        *          <li>asterisk (*) and question mark (?) wildcards are permitted to represent zero or more and
        *              exactly one wildcard characters, respectively
        *          <li>files specified by the primary {@code filespec} which are not matched by the ignore list
        *              are scanned as they would be in {@code filespec} mode.
        *       </ul>
        * </ul>
        *
        * @param    args
        *           List of command line arguments.
        * @param    modes
        *           Valid run modes.
        * @param    help
        *           Help text for syntax display.
        *
        * @return   The details of this job.
        */
       protected static JobDefinition processCommandLine(String[] args, Map<String, RunMode> modes, String[] help)
       {
          // option defaults
          ListType type         = ListType.CMD_LINE;
          boolean  recursion    = true;
          boolean  abortOnError = false;
          boolean  dryRun       = false;
          int      debug        = MSG_STATUS;
          int      idx          = 0;
          int      threads      = 1;
          boolean  incremental  = false;
          List<String> cfgProfiles = new ArrayList<>();

          // requires (mode + directory + filespec) OR (mode and a filelist) OR (mode), with the file-set read from
          // p2j.cfg.xml
          if (args.length < 1)
          {
             syntax(help, null, -1);
          }

          // process options
          while (args[idx].startsWith("-"))
          {
             String opts = args[idx].substring(1).toLowerCase(); // drop the '-'
             idx++;

             if (opts.startsWith("p"))
             {
                // this must be a single option token because the configuration PROFILE name might contain
                // characters which interfere with other options
                cfgProfiles.add(opts.substring(1)); // drop the 'p'
                continue;
             }

             // the next options might all be concatenated after a single "-" prefix but this practice is
             // obsolete and not recommended. Use each option independently.
             if (opts.indexOf('i') != -1)
             {
                incremental = true;
             }
             if (opts.indexOf('s') != -1)
             {
                type = ListType.FILESPEC;
             }
             if (opts.indexOf('f') != -1)
             {
                type = ListType.WHITELIST;
             }
             if (opts.indexOf('x') != -1)
             {
                type = ListType.BLACKLIST;
             }
             if (opts.indexOf('z') != -1)
             {
                type = ListType.FILESET;
             }
             if (opts.indexOf('n') != -1)
             {
                recursion = false;
             }
             if (opts.indexOf('a') != -1)
             {
                abortOnError = true;
             }
             if (opts.indexOf('l') != -1)
             {
                dryRun = true;
             }

             int dbg = opts.indexOf('d');

             if (dbg != -1)
             {
                // get level (set to space char if of out-of-bounds to avoid exception)
                char level = (dbg + 1 < opts.length() ? opts.charAt(dbg + 1) : ' ');

                if (level == ' ' || !Character.isDigit(level))
                {
                   syntax(help, "Missing numeric debug level character following D option.", -2);
                }
                else
                {
                   debug = Character.digit(level, 10);

                   if (debug < MSG_NONE || debug > MSG_TRACE)
                   {
                      syntax(help, "Debug level must be between " + MSG_NONE + " and " + MSG_TRACE + ".", -3);
                   }
                }
             }

             int tpos = opts.indexOf('t');

             if (tpos != -1)
             {
                int start = tpos + 1;
                int next = start;
                int len = opts.length();

                while (next < len && Character.isDigit(opts.charAt(next)))
                {
                   next++;
                }

                boolean fail = next == start;
                if (!fail)
                {
                   try
                   {
                      threads = Integer.parseInt(opts.substring(start, next));
                   }
                   catch (NumberFormatException exc)
                   {
                      // should not be thrown, since we pre-validated characters as digits
                      fail = true;
                   }
                }

                if (fail)
                {
                   syntax(help, "Missing number of threads following T option.", -4);
                }
             }
          }

          // process the mode argument
          String  key = args[idx].toUpperCase();
          RunMode run = interpretRunMode(key, modes, help);
          idx++;

          String transform = null;

          // user-defined rule-set (TODO: hard coding here is should be replaced with a more generic approach)
          if (run.middle && key.contains("TR"))
          {
             if (args.length <= idx)
             {
                syntax(help, "Missing user-defined rule-set parameter.", -6);
             }

             transform = args[idx];
             idx++;
          }

          // configuration profiles (without schema) need to be loaded now, as the file-sets are required to be
          // known bellow.
          Configuration.loadConfigProfiles(cfgProfiles);

          FileList files = null;

          // at this point idx points to the next arg after the options, this
          // must be one of our various forms of filename input
          switch (type)
          {
             case CMD_LINE:
                if (args.length <= idx && Configuration.getFileList() != null)
                {
                   // no explicit command-line files are provided and p2j.cfg.xml has configured file-set
                   files = Configuration.getFileList();
                }

                if (files == null)
                {
                   // there must be at least 1 arg left
                   if (args.length <= idx)
                   {
                      syntax(help, "Missing file list parameter.", -7);
                   }

                   int      len      = args.length;
                   String[] filelist = new String[len - idx];

                   if (len - idx >= 0)
                   {
                      System.arraycopy(args, idx, filelist, 0, len - idx);
                   }

                   files = new ExplicitFileList(filelist);
                }

                break;

             case WHITELIST:
                // there must be exactly 1 arg left
                if ((args.length - idx) != 1)
                {
                   syntax(help, "Missing custom file list parameter.", -8);
                }

                files = FileListFactory.createExplicit(args[idx]);
                break;

             case FILESPEC:
                // there must be exactly 2 args left
                if ((args.length - idx) != 2)
                {
                   syntax(help, "Missing/incomplete filespec parameter.", -9);
                }

                files = new FileSpecList(new File(args[idx]), args[idx + 1], recursion);
                break;

             case BLACKLIST:
                // there must be exactly 3 args left: starting directory, filespec, ignore filename
                if ((args.length - idx) != 3)
                {
                   syntax(help, "Missing/incomplete filespec and exclude list parameters.", -10);
                }

                // TODO: we could set case-sensitivity from the legacy system setting, which would allow us to
                //       read blacklists made with inputs on case-insensitive systems
                files = FileListFactory.createBlacklist(args[idx],
                                                        args[idx + 1],
                                                        recursion,
                                                        false,
                                                        args[idx + 2]);
                break;
             case FILESET:
                if ((args.length - idx) != 1)
                {
                   syntax(help, "Missing file set parameter.", -11);
                }

                FileSet fileSet = FileListFactory.loadFileSet(args[idx]);
                if (fileSet != null)
                {
                   try
                   {
                      files = FileListFactory.processFileSet(fileSet, Configuration.isCaseSensitive());
                   }
                   catch (IOException e)
                   {
                      LOG.log(Level.SEVERE, "Processing file set:", e);
                      return null;
                   }
                }

                break;
          }

          if (files == null)
          {
             return null;
          }

          // if -L option was used, just list files which would be processed to stdout, return null
          if (dryRun)
          {
             LOG.log(Level.CONFIG, "Dry run;  listing source files only...");
             for (File file : files.list())
             {
                LOG.log(Level.CONFIG, file.toString());
             }

             return null;
          }

          return new JobDefinition(run, transform, files, cfgProfiles, threads, abortOnError, incremental, debug);
       }

       /**
        * Read the class hierarchy from the AST associated with the given filename and store the state in
        * the given maps.
        *
        * @param    mgr
        *           AST manager.
        * @param    cls2file
        *           Map of qualified class names to file names.
        * @param    clsHierarchy
        *           Map of qualified class names to a set of subclasses.
        * @param    file
        *           The file name to process.
        */
       private static void processOODefinition(AstManager               mgr,
                                               Map<String, String>      cls2file,
                                               Map<String, Set<String>> clsHierarchy,
                                               String                   file)
       {
          // temporarily load this AST and get:
          // - the qualified class name
          // - the super class (single inherits for a class) or super interfaces (multiple-inherits for
          //   interface)
          // - all implemented interfaces (which we treat the same as a superclass) for a class

          Aast ast    = mgr.loadTree(file);
          Aast def    = ast.getImmediateChild(ProgressParserTokenTypes.CLASS_DEF, null);
          Aast header = null;

          if (def == null)
          {
             def = ast.getImmediateChild(ProgressParserTokenTypes.INTERFACE_DEF, null);

             if (def == null)
             {
                // must be an enum, nothing to do
                return;
             }
             else
             {
                header = def.getImmediateChild(ProgressParserTokenTypes.KW_INTERFAC, null);
             }
          }
          else
          {
             header = def.getImmediateChild(ProgressParserTokenTypes.KW_CLASS, null);
          }

          List<String> supers = new LinkedList<String>();

          if (header.downPath(ProgressParserTokenTypes.KW_INHERITS, ProgressParserTokenTypes.CLASS_NAME))
          {
             Aast ref = header.getImmediateChild(ProgressParserTokenTypes.KW_INHERITS, null);

             processClassNames(ref, clsHierarchy, supers);
          }

          if (header.downPath(ProgressParserTokenTypes.KW_IMPLEMTS, ProgressParserTokenTypes.CLASS_NAME))
          {
             Aast ref2 = header.getImmediateChild(ProgressParserTokenTypes.KW_IMPLEMTS, null);

             processClassNames(ref2, clsHierarchy, supers);
          }

          if (supers.isEmpty())
          {
             supers.add(ROOT_CLASS);
          }

          Aast   sym   = header.getImmediateChild(ProgressParserTokenTypes.SYMBOL, null);
          String qname = sym.getText().toLowerCase();

          cls2file.put(qname, file);

          // link the qname into the set for every super class/super interface/implemented interface
          for (String superClass : supers)
          {
             Set<String> subClasses = clsHierarchy.get(superClass);

             // create a new set if the superclass hasn't been processed yet
             if (subClasses == null)
             {
                clsHierarchy.put(superClass, subClasses = new TreeSet<>());
             }

             // Progress.Lang.Object will recurse endlessly if we try to make it a subclass of itself
             if (!superClass.toLowerCase().equals(qname))
             {
                subClasses.add(qname);
             }
          }
       }

       /**
        * Process the given AST node for all {@code CLASS_NAME} child nodes.  For each child node, read
        * the qualified class name and add it to the given list of super classes.  For built-in classes
        * link them to the root class even if they are not direct children.
        *
        * @param    ref
        *           The {@code KW_INHERITS} or {@code KW_IMPLEMTS} node which has one or more class name
        *           children.
        * @param    clsHierarchy
        *           The map of subclasses for each parent class.
        * @param    supers
        *           The list to which to add any inherited super class, inherited interfaces and any
        *           implemented interfaces.
        */
       private static void processClassNames(Aast                     ref,
                                             Map<String, Set<String>> clsHierarchy,
                                             List<String>             supers)
       {
          Aast child = ref.getImmediateChild(ProgressParserTokenTypes.CLASS_NAME, null);

          while (child != null)
          {
             Boolean builtinCls = (Boolean) child.getAnnotation("builtin-cls");
             String  superClass = (String) child.getAnnotation("qualified");
             superClass = superClass.toLowerCase();

             supers.add(superClass);

             if (builtinCls != null && builtinCls.booleanValue() && !ROOT_CLASS.equals(superClass))
             {
                // for all builtin classes, add them to the 'progress.lang.object' sub-classes
                Set<String> subClasses = clsHierarchy.get(ROOT_CLASS);

                if (subClasses == null)
                {
                   clsHierarchy.put(ROOT_CLASS, subClasses = new TreeSet<>());
                }

                subClasses.add(superClass);
             }

             child = child.getImmediateChild(ProgressParserTokenTypes.CLASS_NAME, child);
          }
       }

       /**
        * Sort all legacy classes in the specified sources, by computed the class hierarchy and
        * performing a BFS walk starting from <code>Progress.Lang.Object</code>.  Interfaces are not
        * included in this sort.
        *
        * @param    sources
        *           The sources to sort.
        *
        * @return   The sources, where classes are at the beginning of the list, in BFS order, with
        *           the rest of the files following it, in alphabetical order.
        */
       private static List<String> sortLegacyClasses(Set<String> sources)
       {
          AstManager mgr = AstManager.get();

          Map<String, String>      cls2file     = new HashMap<>();
          Map<String, Set<String>> clsHierarchy = new HashMap<>();

          for (String file : sources)
          {
             if (file.endsWith(".cls.ast"))
             {
                processOODefinition(mgr, cls2file, clsHierarchy, file);
             }
          }

          List<String> bfs   = walkHierarchy(clsHierarchy, ROOT_CLASS);
          List<String> files = new ArrayList<>();

          for (String cls : bfs)
          {
             String file = cls2file.get(cls);

             if (file != null)
             {
                // may be null in case of skeleton classes
                files.add(file);
             }
          }

          return files;
       }

       /**
        * Perform a BFS walk of the class hierarchy.
        *
        * @param    hierarchy
        *           A map of classes to their direct sub-classes.
        * @param    parent
        *           The root class to start the walk.
        *
        * @return   The class list, in BFS order.
        */
       private static List<String> walkHierarchy(Map<String, Set<String>> hierarchy, String parent)
       {
          parent = parent.toLowerCase();

          Set<String> subClasses = hierarchy.get(parent);

          if (subClasses == null)
          {
             return Collections.emptyList();
          }

          List<String> res = new ArrayList<>();
          res.addAll(subClasses);

          for (String subCls : subClasses)
          {
             res.addAll(walkHierarchy(hierarchy, subCls));
          }

          return res;
       }
    }