# SchemaLoader.java

    /*
    ** Module   : SchemaLoader.java
    ** Abstract : Loads schema namespaces from persistent storage (XML)
    **
    ** Copyright (c) 2004-2024, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ----------------------------------Description-----------------------------------
    ** 001 ECF 20041216   @19183 Created initial version.
    ** 002 ECF 20050118   @19338 Updated calls to SchemaDictionary schema
    **                           loading methods whose signatures have
    **                           changed.
    ** 003 ECF 20050207   @19638 Adapted to load schema metadata information
    **                           along with the regular schema namespace data.
    ** 004 ECF 20050331   @20603 Added support to parse and load into schema
    **                           dictionary all table and field options.
    ** 005 ECF 20050411   @20725 Complete rewrite. Changed class name from
    **                           NamespaceLoader to reflect significantly
    **                           broader functional scope. The loader no
    **                           longer loads a schema-specific XML document
    **                           format. Instead, it uses the AstPersister to
    **                           read its schema data from the generic AST,
    **                           XML format. It also is used standalone to
    **                           import schema data from a DF file using the
    **                           new SchemaParser. The AstPersister is used to
    **                           persist the resulting AST to an XML document.
    ** 006 ECF 20050422   @20845 Register schema ASTs with AST registry.
    ** 007 ECF 20050430   @20989 Exposed methods to load schema AST from file
    **                           and to fetch a schema filename given a
    **                           logical database name.
    ** 008 GES 20070402   @32706 Metadata for preferred databases is marked
    **                           as such to allow ambiguous name conflicts to
    **                           be resolved.
    ** 009 GES 20070629   @34331 Removed terse flag from AST persistence so
    **                           that we retail line/column info in our ASTs.
    **                           Added the "srcfile" annotation to the
    **                           database node in the AST so that the original
    **                           .df filename can easily be found later.
    ** 010 ECF 20070725   @35345 Fixed postprocessing of VIEW-AS option.
    **                           Prevent VIEW-AS child of a VIEW-AS node.
    **                           Added line and column number to error message
    **                           when parsing fails in postprocessing step.
    ** 011 GES 20071030   @35896 Added safety code to allow for empty view-as
    **                           clauses (an empty string as contents).
    ** 012 GES 20080308   @37653 Added proper character set processing for
    **                           I18N. This means that we use character-
    **                           oriented support (Readers) instead of
    **                           byte-oriented (Streams).
    ** 013 GES 20080703   @39054 Fixed bug in post-processing removal which
    **                           was changing the tree during the walk (bad
    **                           idea).
    ** 014 ECF 20080704   @39139 Minor memory optimization. Use Boolean
    **                           constants instead of instantiating new
    **                           Boolean objects.
    ** 015 GES 20090429   @42053 Match package and class name changes.
    ** 016 GES 20090515   @42222 Moved to AstManager from AstRegistry/AstPersister.
    ** 017 GES 20090518   @42402 Moved filename normalization here from brainwash().
    ** 018 GES 20100810          Eliminated unnecessary output when validation
    **                           expressions or view-as phrases fail to parse
    **                           properly. Enhanced the error output to make it
    **                           more condensed and more useful.
    ** 019 ECF 20101005          Allow no metadata schema to be configured.
    ** 020 LMR 20110106          Fixes to reflect renaming of load() method in
    **                           Configuration class to getSchemaConfig() and
    **                           removal of its 'key' parameter.
    ** 021 GES 20110628          Moved to common code for preproc options building.
    ** 022 GES 20110831          Honor case-sensitive configuration for the symbol
    **                           resolver.
    ** 023 GES 20111204          Switched propath processing to common code (which
    **                           fixed a bug because this code did not honor the
    **                           source-platform path separator).
    ** 024 ECF 20130125          Adjusted for metadata-related configuration changes.
    ** 025 CA  20131029          All filenames must be normalized.
    ** 026 SVL 20140320          loadAst(String) loads data from a resource in runtime mode.
    ** 027 ECF 20140924          Normalized schema file name, for safer use with non-file-system
    **                           storage.
    ** 028 ECF 20150104          Added support for implicit DICTDB alias for first loaded database.
    ** 029 ECF 20151101          Removed parsing of validation expressions; we don't have the proper
    **                           context to resolve all references at this point; it has to be done
    **                           during 4GL business logic parsing.
    ** 030 GES 20170708          Match changes in the SymbolResolver constructor. At this point we
    **                           don't need silent mode off or on, we just leave it on so that we
    **                           don't have to obtain the actual value from up the stack.
    ** 031 CA  20180507          Added loadNonDefaults, to load non-default schemas.
    ** 032 CA  20200412          Added incremental conversion support.
    ** 033 OM  20191010          Minor logging change.
    ** 034 IAS 20200624          Fixed conversion of the VIEW-AS field property.
    **     GES 20200629          Fixed regression in VIEW-AS processing (embedded quotes and escape sequences
    **                           were not encoded properly in the DMO.
    **     GES 20200720          Honor a source-charset configuration setting when reading the df file(s).
    **     IAS 20200809          Improved conversion of the DESCRIPTION field property.
    **     OM  20210908          Used SchemaDictionary.TEMP_TABLE_DB instead of hardcoded constant.
    **                           Marked imported .dict files with dedicated flag.
    **     OM  20211122          Added hints-based support for 'unloading' schema.
    **     OM  20220330          Moved database conversion artifacts to ${cvtpath} folder.
    **     GES 20220404          Added debug logging.
    **     CA  20220501          Each profile has the same structure as the main 'global' config.  Allow multiple
    **                           profiles to be ran at once, with the conversion switching the state between
    **                           profiles, when a resource (like a file or namespace) is being processed.  Only
    **                           front phase is supported at this time.
    **     ECF 20230127          Added support for mutable schemas, potentially with no corresponding DF files.
    ** 035 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
    ** 036 OM  20240410          mergeMetadata() copies the _meta tables even if the primary database is empty.
    **                           Skip loading tables from mutable databases, they will be loaded on request.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.schema;

    import java.io.*;
    import java.nio.charset.*;
    import java.util.*;
    import java.util.logging.*;

    import com.goldencode.ast.*;
    import com.goldencode.p2j.cfg.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.preproc.*;
    import com.goldencode.p2j.uast.*;
    import com.goldencode.p2j.util.*;

    /**
     * Manages the import and loading of schema information, to make this data
     * available to the {@link SchemaDictionary}. This service is provided in
     * two phases:
     * <ol>
     * <li>Schema data is imported from a set of Progress data dictionary export
     *     (<code>*.df</code>) files. For each file imported, an AST is created.
     *     The AST structure is largely compatible with the source code ASTs
     *     generated by the Progress parser. This information is persisted to
     *     an XML file. This phase is driven from the command line (see {@link
     *     #main}).
     * <li>Schema data is loaded from the XML-formatted ASTs persisted in the
     *     previous phase, into the schema dictionary for use during source code
     *     parsing by the Progress parser. This phase is driven programmatically
     *     by the schema dictionary.
     * </ol>
     *
     * <h3>Schema Import</h3>
     * Importing schema information from a DF file actually occurs in two steps,
     * because the second step is dependent upon the output from the first step:
     * <ol>
     * <li><strong>Parsing the DF file</strong><br>
     *     One DF file must be designated in the P2J configuration as the
     *     metadata file (i.e., the schema which describes the other schemas).
     *     The {@link SchemaParser} is used to parse the schema's metadata DF
     *     file into an AST. Then, each logical database's schema is parsed into
     *     its own AST. These ASTs are mostly complete, except that certain
     *     directives in the DF file contain information which can only be fully
     *     parsed once the structure of the database's schema is available.
     *     These are left as unprocessed strings in the first-pass AST.
     * <li><strong>AST Postprocessing</strong><br>
     *     In this step, the directives left unprocessed in the previous step
     *     are processed into their own ASTs, which are then grafted onto the
     *     tree, so as to replace the AST node which previously contained the
     *     unprocessed string version. The strings are actually snippets of
     *     Progress source code, which must be run through the Progress
     *     preprocessor (to expand any preprocessor statements), and then through
     *     the Progress parser (to create a true AST from the snippet of code).
     *     However, as the snippets often contain references to schema entities,
     *     such as tables and fields, the Progress parser requires a minimally
     *     working schema dictionary to resolve these symbols. Hence the need
     *     for two steps. A copy of the AST for the current logical database
     *     (created in the previous step) is merged with the metadata AST,
     *     such that the metadata schema elements appear to be elements of the
     *     logical database itself (they are, in fact, common elements of every
     *     logical database and are referenced as such in Progress source code).
     *     The merged AST is then loaded into a schema dictionary instance,
     *     which is embedded into a symbol resolver, which is in turn used by
     *     the Progress parser when parsing the code snippets. If a code snippet
     *     cannot be preprocessed or parsed successfully, the original string
     *     AST node is left in place; otherwise, it is replaced by the expanded
     *     AST. These changes are made to the original, logical database AST,
     *     not to the copy which was merged with the schema metadata AST (the
     *     latter is needed only to initialize the temporary schema dictionary).
     * </ol>
     *
     * The only directive deferred to the AST postprocessing pass is <code>VIEW-AS</code>
     * (phrases which determine a field's default representation in the UI).
     * We previously attempted to postprocess <code>VALEXP</code> as well, but this was too
     * early, as some such expressions required the context of the business logic in which
     * they are applied. Processing of these has since been pushed into the Progress parser.
     * <p>
     * After the two-step import is complete, the fixed-up, logical database AST
     * is persisted to XML in a terse format using the general purpose {@link AstManager} class.
     *
     * <h3>Schema Dictionary Loading</h3>
     * The schema dictionary invokes either {@link #loadDefaults} to load all
     * default databases, or {@link #loadSchema} to retrieve an individual
     * database. In either case, the metadata AST is always loaded and merged
     * with the logical database ASTs being loaded. The {@link SchemaConfig}
     * class is used to read the current configuration to determine which
     * database represents the metadata schema, as well as to determine which
     * logical databases should be loaded by default.
     * <p>
     * When loading the schema dictionary, the ASTs are always read from the
     * XML documents created by a previous import. This is a practical necessity,
     * since the import process is considerably time consuming.
     *
     * @see  SchemaConfig
     * @see  SchemaDictionary
     * @see  SchemaParser
     * @see  com.goldencode.p2j.preproc.Preprocessor
     * @see  com.goldencode.p2j.uast.ProgressParser
     */
    public class SchemaLoader
    implements SchemaParserTokenTypes
    {
       /** Footer content for an (otherwise) empty DF file */
       private static final String emptyDfFooter =
          "\n" +
          ".\n" +
          "PSC\n" +
          "cpstream=UTF-8\n" +
          ".\n" +
          "0000000000\n";

       /** Logger */
       private static final ConversionStatus LOG = ConversionStatus.get(SchemaLoader.class);

       /** Schema configuration object */
       private final SchemaConfig config;

       /** Flag indicating quiet mode */
       private boolean quietMode = false;

       static
       {
          try
          {
             String name = Configuration.getParameter("registry");
             AstManager.initialize(() -> new XmlFilePlugin(name, Configuration.isImportMode()));
          }
          catch (AstException ae)
          {
             // TODO: log this? get ready for null pointer exceptions later
          }
       }

       /**
        * Default constructor. Loads schema configuration information.
        *
        * @throws  SchemaException
        *          if an error occurs reading configuration data.
        */
       public SchemaLoader()
       throws SchemaException
       {
          config = Configuration.getSchemaConfig();
       }

       /**
        * Imports all DF files defined in the configuration, generates an AST
        * for each schema so defined, and persists that AST to a file defined
        * by the configuration.
        *
        * @throws  SchemaException
        *          if any error occurs while importing the schema information
        *          or persisting the ASTs.
        */
       public void importAll()
       throws SchemaException
       {
          String[] paths = AstGenerator.getPropath(null, null);

          // Import and postprocess the metadata schema.
          String metaName = config.getMetadata().getName();
          Aast metaAst = null;
          if (metaName != null)
          {
             metaAst = importSchema(metaName, null, paths);
          }
          else
          {
             System.out.println("WARNING:  no schema metadata configured");
          }
          Set<String> profiles = new HashSet<>();

          Configuration cfg = Configuration.getInstance();

          // Import and postprocess each non-metadata schema.
          Iterator<String> iter = config.databases(cfg.isMultiProfile());
          while (iter.hasNext())
          {
             String name = iter.next();

             if (cfg.isMultiProfile())
             {
                SchemaException[] exc = new SchemaException[1];
                cfg.withDbProfile(name, (profile) ->
                {
                   try
                   {
                      String[] profilePaths = AstGenerator.getPropath(null, null);

                      // Import and postprocess the metadata schema.
                      String profileMetaName = config.getMetadata().getName();
                      Aast profileMetaAst = null;
                      if (profileMetaName != null)
                      {
                         if (!profiles.contains(profile))
                         {
                            profileMetaAst = importSchema(profileMetaName, null, profilePaths);
                         }
                      }
                      else
                      {
                         System.out.println("WARNING:  no schema metadata configured");
                      }

                      if (profileMetaName == null || !profileMetaName.equals(name))
                      {
                         importSchema(name, profileMetaAst, profilePaths);
                      }
                   }
                   catch (SchemaException e)
                   {
                      exc[0] = e;
                   }
                });

                if (exc[0] != null)
                {
                   throw exc[0];
                }
                continue;
             }

             if (metaName == null || !metaName.equals(name))
             {
                importSchema(name, metaAst, paths);
             }
          }

          try
          {
             // Save the registry's state.
             AstManager.get().save();
          }
          catch (AstException exc)
          {
             throw new SchemaException("Error saving AST registry", exc);
          }
       }

       /**
        * Import information for an individual, logical database schema and
        * persist the resulting AST in a file defined by the configuration.
        *
        * @param   name
        *          Name of the logical database to be imported and persisted.
        * @param   metaAst
        *          AST representing the schema metadata information. This data
        *          is merged with a copy of the logical database's schema AST
        *          in order to initialize a temporary schema dictionary which
        *          is needed to parse snippets of Progress code embedded in
        *          the database's schema information.
        * @param   paths
        *          The individual file paths which comprise the PROPATH defined
        *          in the configuration. This information is needed by the
        *          Progress preprocessor when preprocessing snippets of Progress
        *          code embedded in the database's schema information.
        *
        * @throws  SchemaException
        *          if any error occurs importing the schema information or
        *          persisting the AST.
        */
       private Aast importSchema(String name, Aast metaAst, String[] paths)
       throws SchemaException
       {
          Aast              root = null;
          InputStreamReader isr  = null;
          BufferedReader    in   = null;

          try
          {
             // get DF file; if the schema is mutable, it may not exist, and this is valid
             File file = config.getImportFile(name);
             boolean useDfFile = file != null && file.exists() && file.isFile();
             boolean mutable = config.isMutable(name);

             if (!quietMode)
             {
                if (useDfFile)
                {
                   System.out.println("Importing '" + file.getName() + "' for schema '" + name + "'...");
                }
                else if (mutable)
                {
                   System.out.println("Processing mutable schema '" + name + "'...");
                }
             }

             boolean caseSens = Configuration.getParameter("case-sensitive", true);
             String  encoding = Configuration.getParameter("source-charset");
             Charset charset  = (encoding == null) ? Charset.defaultCharset() : Charset.forName(encoding);

             // Create "bootstrap" resolver (i.e., no schema dictionary).
             SymbolResolver resolver = new SymbolResolver(false, caseSens, null, true, false, null);

             // Set up Progress lexer in schema mode.
             if (useDfFile)
             {
                // the DF file exists, so read it
                isr = new InputStreamReader(new FileInputStream(file), charset);
             }
             else if (mutable)
             {
                // it is valid for a mutable schema to have no DF file, so set up a minimal stream for the lexer
                isr = new InputStreamReader(new ByteArrayInputStream(emptyDfFooter.getBytes()));
             }
             in  = new BufferedReader(isr);
             ProgressLexer lexer = new ProgressLexer(in, resolver);
             lexer.activateSchemaProcessing();

             // Parse data and create a tree
             SchemaParser parser = new SchemaParser(lexer, true);
             parser.schema();

             File schemaFile = config.getSchemaFile(name, AstGenerator.DICT_POSTFIX);

             String filename = schemaFile.getAbsolutePath();
             String normal   = Configuration.normalizeFilename(filename);

             // Get the root AST, register it, and fix it up.
             root = (Aast) parser.getAST();
             root.brainwash(normal);
             AstManager.get().setFlag(root.getId(), AstGenerator.FLAG_DICT);

             // Replace the unknown database name with the actual name.
             root.setText(name);

             // Store the original .df filename as an annotation in the database
             // node.
             String actual  = file.getCanonicalPath();
             String srcfile = Configuration.normalizeFilename(actual);
             root.putAnnotation("srcfile", srcfile);
          }
          catch (Exception exc)
          {
             throw new SchemaException("Error importing schema data: " + name, exc);
          }
          finally
          {
             // Clean up stream.
             try
             {
                if (in != null)
                {
                   in.close();
                }
                else if (isr != null)
                {
                   isr.close();
                }
             }
             catch (IOException exc)
             {
                // don't care
             }
          }

          // Postprocess the first pass result to resolve inlined Progress code.
          root = postProcessImport(root, metaAst, paths);

          // Persist the postprocessed AST.
          persistSchema(root);

          return root;
       }

       /**
        * Postprocess the DF file directives which include Progress code snippets
        * in a <code>STRING</code> AST, and replace that <code>STRING</code> AST
        * with an expanded AST which represents the fully parsed source code for
        * those directives.
        * <p>
        * The directive affected are:
        * <ul>
        * <li><code>VALEXP</code> - field and table validation expressions
        * <li><code>VIEW-AS</code> - view-as phrases which determine a field's
        *     default representation in the UI
        * </ul>
        *
        * <strong>Note:</strong> it appears to be legal to have invalid (i.e.,
        * uncompilable) Progress source code in these directives. Thus, during
        * postprocessing of these directives, the Progress preprocessor may
        * encounter includes it cannot resolve, and the Progress parser may
        * report unexpected tokens. Not all of these errors result in an
        * exception, so to be aware of problems during postprocessing, it is
        * critical to review the console output generated by this class.
        *
        * @param   data
        *          AST of the logical database as generated by the {@link
        *          SchemaParser}. Contains the <code>STRING</code> ASTs to be
        *          replaced.
        * @param   meta
        *          AST representing the schema metadata information. This data
        *          is merged with a copy of the logical database's schema AST
        *          in order to initialize a temporary schema dictionary which
        *          is needed to parse snippets of Progress code embedded in
        *          the database's schema information.
        * @param   paths
        *          The individual file paths which comprise the PROPATH defined
        *          in the configuration. This information is needed by the
        *          Progress preprocessor when preprocessing snippets of Progress
        *          code embedded in the database's schema information.
        *
        * @throws  SchemaException
        *          if any error occurs importing the schema information or
        *          persisting the AST.
        */
       private Aast postProcessImport(Aast data, Aast meta, String[] paths)
       throws SchemaException
       {
          // Merge metadata schema into data schema for use with the schema dictionary for symbol resolution
          // during parsing of inlined code. Make a copy of the data AST so that we can persist the non-merged
          // original when postprocessing is done. While this makes for a big footprint, imports are an offline
          // process that rarely happens, so it's OK
          if (meta != null)
          {
             meta = meta.duplicate(null);
          }
          Aast merged = mergeMetadata(data, meta, true, true);

          // Set up the symbol resolver using the schema information imported
          // from the DF file to handle schema references.
          SymbolResolver resolver = new SymbolResolver(false, true, null, true, false, null);
          SchemaDictionary dictionary = new SchemaDictionary(merged);
          resolver.setSchemaDictionary(dictionary);

          // This is the list of any ASTs that need to be removed from the tree.
          // We cannot remove inside the loop below that is iterating on the tree
          // since that would truncate the iteration.  So this allows the deletes
          // to be deferred.
          ArrayList<Aast> deletes = new ArrayList<>();

          // Walk the AST and postprocess inlined Progress code. This is only
          // necessary for a subset of the directives.
          dictionary.addScope();
          Iterator<Aast> iter = data.iterator();
          while (iter.hasNext())
          {
             Aast next = iter.next();
             switch (next.getType())
             {
                case TABLE:
                   dictionary.removeScope();
                   dictionary.addScope();
                   dictionary.promoteTable(next.getText());
                   break;

                case KW_VIEW_AS:
                   Aast del = replaceAst(next, resolver, paths);

                   if (del != null)
                   {
                      deletes.add(del);
                   }
                   break;
                case DESCRIPTION:
                   String descrText = character.encodeToJavaSource(next.getText().replaceAll("&quot;", "\""));
                   next.putAnnotation("descr", descrText);
                   break;
             }
          }

          dictionary.removeScope();

          // Process deferred deletes.
          for (Aast next : deletes)
          {
             next.remove();
          }

          return data;
       }

       /**
        * Persists a schema AST to the file defined in the configuration for
        * the logical database represented by <code>ast</code>.
        *
        * @param   ast
        *          The schema information AST to be persisted.
        *
        * @throws  SchemaException
        *          if any error occurs persisting the file.
        */
       private void persistSchema(Aast ast)
       throws SchemaException
       {
          String name = ast.getText();
          try
          {
             File schemaFile = config.getSchemaFile(name, AstGenerator.DICT_POSTFIX);
             String normalized = Configuration.normalizeFilename(schemaFile.getCanonicalPath());

             AstManager.get().saveTree(ast, normalized, false);

             if (!quietMode)
             {
                System.out.println("Persisted schema '" + name + "' to '" + schemaFile.getName() + "'");
             }
          }
          catch (Exception exc)
          {
             throw new SchemaException("Error persisting schema:  " + name, exc);
          }
       }

       /**
        * Turn on/off verbose console output.
        *
        * @param   quietMode
        *          <code>true</code> to report progress to the console verbosely;
        *          <code>false</code> to suppress status messages.
        */
       public void setQuietMode(boolean quietMode)
       {
          this.quietMode = quietMode;
       }

       /**
        * Load schema information for schemas specified by the configuration as default schemas, into
        * the specified dictionary. The first non-metadata database to be loaded is assigned the
        * implicit DICTDB alias.
        *
        * @param   dictionary
        *          Dictionary into which databases will be loaded.
        *
        * @throws  SchemaException
        *          if any error occurs loading a schema.
        */
       public void loadDefaults(SchemaDictionary dictionary)
       throws SchemaException
       {
          boolean dictdbSet = false;
          String metaName = config.getMetadata().getName().toLowerCase();

          Iterator<String> iter = config.defaultDatabases();
          while (iter.hasNext())
          {
             String dbName = iter.next();
             try
             {
                if (config.isMutable(dbName))
                {
                   continue; // do not load mutable databases at this time, even if 'default' attribute was set
                }
             }
             catch (ConfigurationException e)
             {
                throw new SchemaException(e);
             }

             Aast ast = loadSchema(dbName);

             // the first non-metadata database to be loaded is assigned the DICTDB alias
             boolean dictdb = false;
             if (!dictdbSet && !metaName.equals(dbName.toLowerCase()))
             {
                dictdb = true;
                dictdbSet = true;
             }

             dictionary.loadFromAst(ast, dictdb);
          }
       }

       /**
        * Load all non-default schemas into the dictionary.
        *
        * @param   dictionary
        *          Dictionary into which databases will be loaded.
        *
        * @throws  SchemaException
        *          if any error occurs loading source file-specific schema information.
        */
       public void loadNonDefaults(SchemaDictionary dictionary)
       throws SchemaException
       {
          Configuration cfg = Configuration.getInstance();

          Set<String> defaults = new HashSet<>();
          config.defaultDatabases().forEachRemaining(defaults::add);
          String metaSchema = config.getMetadata().getName();

          Iterator<String> iter = config.databases(cfg.isMultiProfile());
          while (iter.hasNext())
          {
             String dbName = iter.next();

             if (cfg.isMultiProfile())
             {
                SchemaException[] exc = new SchemaException[1];

                cfg.withDbProfile(dbName, (profile) ->
                {
                   defaults.clear();
                   config.defaultDatabases().forEachRemaining(defaults::add);
                   String profileMetaSchema = config.getMetadata().getName();

                   if (!defaults.contains(dbName) && !profileMetaSchema.equals(dbName))
                   {
                      try
                      {
                         Aast ast = loadSchema(dbName);
                         dictionary.loadFromAst(ast, false);
                      }
                      catch (SchemaException e)
                      {
                         exc[0] = e;
                      }
                   }
                });

                if (exc[0] != null)
                {
                   throw exc[0];
                }

                continue;
             }

             if (!defaults.contains(dbName) && !metaSchema.equals(dbName))
             {
                Aast ast = loadSchema(dbName);

                dictionary.loadFromAst(ast, false);
             }
          }
       }

       /**
        * Load database schema information from a persisted AST associated with
        * the specified, logical database name. Loads both the schema data
        * associated with the database and the schema metadata which is
        * associated with all databases in the current version of Progress.
        *
        * @param   database
        *          Name of the database whose schema namespace data is to be
        *          loaded. This name must match a namespace defined in the schema
        *          configuration and may not be <code>null</code>.
        *
        * @return  An AST which contains the schema information for the logical
        *          database specified, merged with the AST containing metadata
        *          for the schema.
        *
        * @throws  SchemaException
        *          if <code>database</code> is <code>null</code>, or
        *          if no namespace information has been configured for the
        *          specified database name, or if an I/O or XML parsing error
        *          occurs reading the namespace data.
        */
       public Aast loadSchema(String database)
       throws SchemaException
       {
          if (database == null)
          {
             throw new SchemaException("Database name may not be null");
          }

          // Load the schema metadata.
          String metaName = config.getMetadata().getName();
          Aast metaAst = null;
          if (metaName != null)
          {
             metaAst = loadAst(metaName);
          }

          // Load the schema data.
          Aast dataAst = loadAst(database);

          // Merge the metadata into the primary database AST. Merge in place (no copy).
          mergeMetadata(dataAst, metaAst, config.isPreferred(database), false);

          return dataAst;
       }

       /**
        * Load the specified database AST from persistence.
        *
        * @param   file
        *          File containing the database AST to be loaded.
        *
        * @return  AST representing the schema.
        *
        * @throws  SchemaException
        *          if any error occurs reading the AST from persistence.
        */
       public Aast loadAst(File file)
       throws SchemaException
       {
          try
          {
             // Read AST from persistence.
             String fname = file.getCanonicalPath();
             fname = Configuration.normalizeFilename(fname);

             return AstManager.get().loadTree(fname);
          }
          catch (Exception exc)
          {
             throw new SchemaException("Error loading schema data", exc);
          }
       }

       /**
        * Load the specified database AST from persistence. The file which
        * contains the persisted AST is read from the schema configuration.
        *
        * @param   name
        *          Logical name of the database whose schema is to be loaded.
        *
        * @return  AST representing the schema of the requested database.
        *
        * @throws  SchemaException
        *          if any error occurs reading the AST from persistence.
        */
       private Aast loadAst(String name)
       throws SchemaException
       {
          try
          {
             if (Configuration.isRuntimeConfig() || Configuration.isImportMode())
             {
                String resourceName = config.getSchemaResource(name, AstGenerator.DICT_POSTFIX);
                if (resourceName != null)
                {
                   return AstManager.get().loadTree(resourceName);
                }
                else
                {
                   throw new ConfigurationException("Cannot find resource for database " + name);
                }
             }
             else
             {
                File file = config.getSchemaFile(name, AstGenerator.DICT_POSTFIX);
                return loadAst(file);
             }
          }
          catch (Exception exc)
          {
             throw new SchemaException("Error loading schema data", exc);
          }
       }

       /**
        * Merge AST <code>meta</code> into AST <code>data</code>, or into a copy
        * of AST <code>data</code>. Although it can be time consuming to make a
        * copy of the <code>data</code> AST, it is necessary during the import
        * phase to preserve the original, since we want to persist it without
        * the schema metadata.
        * <p>
        * The merge is performed by setting the existing first child of
        * <code>data</code> (or of its copy) as the next sibling of the last
        * first-level child of <code>meta</code>, and the first child of
        * <code>meta</code> as the new first child of <code>data</code>. Then,
        * all first-level children of <code>meta</code> are traversed and their
        * parents set to <code>data</code> (or its copy).
        * <p>
        * Note that the beginning state of <code>meta</code> is never preserved
        * in this algorithm, so the caller should make a copy of <code>meta</code>
        * if such preservation is important.
        *
        * @param   data
        *          AST into which the information in <code>meta</code> will be
        *          merged.
        * @param   meta
        *          AST containing schema metadata to be merged into the
        *          <code>data</code> AST (or a copy thereof). This AST does not
        *          survive the merger, as noted above.
        * @param   prefer
        *          <code>true</code> if the metadata values for this database
        *          should be preferred in any ambiguous name conflict with other
        *          non-preferred databases. This does not affect non-metadata
        *          name lookup.
        * @param   makeCopy
        *          If <code>true</code>, <code>meta</code> is merged into a copy
        *          of <code>data</code>. If <code>false</code>, it is merged
        *          directly into <code>data</code>.
        *
        * @return  The merged AST.
        */
       private Aast mergeMetadata(Aast data, Aast meta, boolean prefer, boolean makeCopy)
       {
          if (meta == null)
          {
             // No copy in this case, since there is no merge to differentiate
             // the merged copy from the original.
             return data;
          }

          if (makeCopy)
          {
             data = data.duplicate(null);
          }

          if (prefer)
          {
             markPreferred(meta);
          }

          Aast firstDataAst = (Aast) data.getFirstChild();
          Aast next = (Aast) meta.getFirstChild();
          if (next != null)
          {
             data.setFirstChild(next);
             Aast last = null;
             while (next != null)
             {
                last = next;
                next.setParent(data);
                next = (Aast) next.getNextSibling();
             }

             if (firstDataAst != null)
             {
                last.setNextSibling(firstDataAst);
             }
          }

          return data;
       }

       /**
        * Set an annotation in every node of this subtree to specify that that
        * each node is to be preferred in ambiguous name conflicts.
        *
        * @param   ast
        *          The subtree to mark.
        */
       private void markPreferred(Aast ast)
       {
          Iterator<Aast> iter = ast.iterator();

          while (iter.hasNext())
          {
             Aast node = iter.next();
             node.putAnnotation("preferred", Boolean.TRUE);
          }
       }

       /**
        * This method is the heart of the DF file import postprocessing step.
        * It is called with an AST which represents one of the directives
        * requiring postprocessing. The immediate and only child of this AST
        * is a string AST containing a snippet of Progress code. This text
        * is extracted and is run through the Progress preprocessor and then the
        * Progress parser. The resulting AST, if any, replaces the string AST
        * child of <code>ast</code>. If the Progress parser produces no AST,
        * an exception is thrown, and the string AST is not replaced. Not all
        * preprocessing and parsing errors result in an exception, however, so
        * it is important to review the console output produced by the
        * preprocessor and parser tools in the course of running this method.
        *
        * @param   ast
        *          AST whose child is the string to be postprocessed and
        *          and replaced.
        * @param   resolver
        *          Object used by the Progress parser to resolve schema and
        *          other symbols found in the Progress code snippets.
        * @param   paths
        *          The individual file paths which comprise the PROPATH defined
        *          in the configuration. This information is needed by the
        *          Progress preprocessor when preprocessing the Progress code
        *          snippets.
        *
        * @return  The node to remove from the tree when the iteration is
        *          complete or <code>null</code> if no node needs to be removed.
        *
        * @throws  SchemaException
        *          if an error occurs while preprocessing or parsing the code
        *          snippets, or if no AST results from this effort, due to
        *          non-fatal errors.
        */
       private Aast replaceAst(Aast ast, SymbolResolver resolver, String[] paths)
       throws SchemaException
       {
          // Return value.
          Aast ret = null;

          // Placeholder for caught exception.
          Exception cause = null;

          // Get the raw text to be preprocessed/parsed, which is stored in the
          // (only) immediate child of the AST.
          Aast child = ast.getImmediateChild(STRING, null);
          String text = child.getText();

          try
          {
             // Preprocess text, in case there are any includes that need to be resolved.
             Options options = AstGenerator.buildOptions(paths, null, null);
             StringWriter out = new StringWriter();
             FileScope fs = new FileScope("<inline>", new StringReader(text));
             new Preprocessor(fs, out, new PrintWriter(System.err), null, options);
             text = out.toString();

             // Empty clause is valid but equivalent to being non-existing so
             // we just remove the original node.
             if (text.length() == 0)
             {
                return ast;
             }

             // Parse the text using the Progress parser.
             StringReader reader = new StringReader(text);
             ProgressLexer lexer = new ProgressLexer(reader, resolver);
             ProgressParser parser = new ProgressParser(lexer, resolver);
             parser.setConsumeError(false);

             int tokType = ast.getType();
             if (tokType == KW_VIEW_AS)
             {
                parser.view_as_phrase(null);
             }

             Aast replacement = (Aast) parser.getAST();

             if (replacement != null)
             {
                // In the case of VIEW-AS, the replacement's top level node is a
                // KW_VIEW_AS, which is also the string expression's parent type,
                // so we must decapitate the replacement and graft its child
                // instead.
                if (tokType == KW_VIEW_AS)
                {
                   // The original text was processed using parseProgressCharLiteral (in schema.g) which means
                   // it was converted into a Java runtime string.  Now we must make sure that embedded quotes
                   // and escape sequences are correct for emitting into source code.
                   String viewAsText = character.encodeToJavaSource(ast.getRightAdjacent().getText());

                   ast.putAnnotation("view-as-text", viewAsText);
                   replacement = replacement.getChildAt(0);
                }

                // Replace the original AST.
                int index = child.getIndexPos();
                child.remove();
                ast.graftAt(replacement, index);
             }
             else
             {
                cause = new NullPointerException("No AST generated by parser");
             }
          }
          catch (Exception exc)
          {
             cause = exc;
          }

          if (cause != null)
          {
             ret = ast;
             LOG.log(Level.WARNING, "Ignoring invalid option " + ast.getSymbolicTokenType() +
                                " [" + ast.getLine() + ":" + ast.getColumn() +
                                "] (error = '" + cause.getMessage() + "'):  " + text);
             LOG.log(Level.FINEST, "", cause);
          }

          return ret;
       }

       /**
        * Command line entry point into this program, used when importing all
        * configured DF files. Requires that the <code>P2J_HOME</code> property
        * be set properly. Accepts one optional (case-insensitive) argument:
        * <code>-q</code> to indicate {@link #setQuietMode quiet mode}.
        *
        * @param   args
        *          0 or 1 argument as noted above.
        */
       public static void main(String[] args)
       {
          try
          {
             SchemaLoader loader = new SchemaLoader();
             if (args.length > 0 && "-q".equalsIgnoreCase(args[0]))
             {
                loader.setQuietMode(true);
             }
             loader.importAll();
          }
          catch (Exception exc)
          {
             LOG.log(Level.SEVERE, "", exc);
          }
       }
    }