# SchemaCheck.java

    /*
    ** Module   : SchemaCheck.java
    ** Abstract : Detects schema changes and reflects them in FWD runtime state.
    **
    ** Copyright (c) 2022-2024, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
    ** 001 ECF 20221001 Created initial version, refactored from BS' changes to DatabaseManager.
    **     BS  20221103 DMOs loading and unloading complete for tables, columns, indexes.
    **     BS  20221104 JavaDocs updated.
    **     ECF 20221207 Fixed package path for generated DMO interface. Renamed some methods.
    **     ECF 20230125 Changed signature and access of run method variant to be used more easily during database
    **                  bootstrap. Prevent concurrent access for any given database. Cleaned up data type code.
    **     ECF 20230127 Fixed some problems with initial value processing.
    ** 002 BS  20230331 Added support for extent fields, DTTZ fields and sequences.
    ** 003 OM  20230227 Fixed index processing in generateSchemaAst().
    ** 004 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    ** 005 CA  20230602 The AST for the index component must have the name (for the field) normalized the same way
    **                  as the table field is normalized (via 'rightTrimLower').
    ** 006 OM  20240318 Added dump-table (mandatory table attribute) when a new table is registered.
    ** 007 OM  20240402 Added proper support for deregistration/dropping of dynamic tables.
    ** 008 OM  20240412 Added sorting direction for index components.
    ** 009 OM  20240507 Added customizations of tables based on the comments/remarks from SQL metadata.
    ** 010 OM  20240516 The SQL tables can be skipped from import if custom comment is set.
    **     OM  20240517 Throw StopConditionException() in the mutable database failed validation.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist;

    import static com.goldencode.p2j.persist.DynamicTablesHelper.*;
    import java.util.*;
    import java.util.concurrent.*;
    import java.util.concurrent.atomic.*;
    import java.util.logging.*;
    import java.util.stream.*;
    import com.goldencode.asm.*;
    import com.goldencode.ast.*;
    import com.goldencode.p2j.convert.ParmType;
    import com.goldencode.p2j.persist.dialect.*;
    import com.goldencode.p2j.persist.meta.*;
    import com.goldencode.p2j.persist.orm.*;
    import com.goldencode.p2j.persist.orm.schema.*;
    import com.goldencode.p2j.persist.orm.schema.element.*;
    import com.goldencode.p2j.schema.*;
    import com.goldencode.p2j.uast.*;
    import com.goldencode.p2j.util.*;
    import antlr.*;
    import com.goldencode.p2j.util.logging.*;

    /**
     * This class uses JDBC metadata to detect external modifications to a mutable database schema and updates
     * the FWD runtime state to reflect such changes.
     */
    public class SchemaCheck
    {
       /** Logger. */
       private static final CentralLogger LOG = CentralLogger.get(SchemaCheck.class);

       /** Constant for SQL NULL text reported by JDBC metadata */
       // TODO: make this more dialect-aware; have only seen this reported by MariaSB so far
       private static final String SQL_NULL = "NULL";

       /** Counter used for generation of unique names for dynamic tables. */
       private static final AtomicLong tableCounter = new AtomicLong(1);

       /** One {@code SchemaCheck} instance per mutable database, on which to synchronize operations */
       private static final ConcurrentMap<Database, SchemaCheck> instances = new ConcurrentHashMap<>();

       /**
        * Delimiters for pairs of [Token: value] in SQL comment. This is a RegExp used in
        * {@link String#split(String)}.
        */
       private static final String DELIMITERS = "\\|";

       /** Database being inspected. */
       private final Database database;

       /** Package into which DMO interface and implementation classes are to be added */
       private final String dmoPackage;

       /** Stored metadata information on dynamic tables mapped by their legacy name. */
       private final Map<String, DmoMeta> knownTables = new HashMap<>();

       /** Token that specifies the legacy name for a table/field/index. */
       private static final String LEGACY_NAME = "LegacyName:";

       /** Token that specifies the dump name of a table. */
       private static final String DUMP_NAME = "DumpName:";

       /** Token that specifies the legacy LABEL attribute. */
       private static final String LABEL = "Label:";

       /** Token that specifies the legacy COLUMN-LABEL attribute. */
       private static final String COLUMN_LABEL = "ColumnLabel:";

       /** Token that specifies the legacy FORMAT attribute. */
       private static final String FORMAT = "Format:";

       /** Token that specifies the legacy CASE-SENSITIVE attribute. */
       private static final String CASE_SENSITIVE = "CaseSensitive:";

       /** Token that specifies the legacy DIGITS attribute, where available. */
       private static final String DIGITS = "Digits:";

       /** Token that specifies the legacy type. */
       private static final String LEGACY_TYPE = "Type:";

       /** Token that specifies the legacy type. */
       public static final String SKIP = "Skip:";

       /** Collection of all known comment tokens. */
       private static final Set<String> COMMENT_TOKENS = new HashSet(){{
             add(CASE_SENSITIVE);
             add(COLUMN_LABEL);
             add(DIGITS);
             add(DUMP_NAME);
             add(FORMAT);
             add(LABEL);
             add(LEGACY_NAME);
             add(LEGACY_TYPE);
             add(SKIP);
       }};

       /**
        * Notify the FWD runtime that a mutable database schema may have been modified. This causes the specified
        * database's schema to be scanned and compared with a previously captured state, for the purpose of
        * updating the FWD runtime's state to match the detected schema changes.
        * <p>
        * This is intended to be a development-time operation and should not be invoked while the target
        * database is in active use, as DMO classes will be added and potentially removed as part of the update.
        *
        * @param   ldbName
        *          Logical name of the database whose schema is meant to have changed.
        */
       public static void run(character ldbName)
       {
          try
          {
             Database database = DatabaseManager.getMutableDatabase(ldbName.toStringMessage());
             run(database, false);
          }
          catch (PersistenceException exc)
          {
             if (LOG.isLoggable(Level.SEVERE))
             {
                LOG.log(Level.SEVERE, "Error running schema check", exc);
             }
             throw new StopConditionException(
                   "Failed to scan mutable database '" + ldbName.toStringMessage() + "'", exc);
          }
       }

       /**
        * Notify the FWD runtime that a mutable database schema may have been modified. This causes the specified
        * database's schema to be scanned and compared with a previously captured state, for the purpose of
        * updating the FWD runtime's state to match the detected schema changes.
        * <p>
        * This is intended to be a development-time operation and should not be invoked while the target
        * database is in active use, as DMO classes will be added and potentially removed as part of the update.
        *
        * @param   database
        *          Helper object which represents physical database whose schema is meant to have changed.
        * @param   bootstrap
        *          {@code true} if invoked during database bootstrap; {@code false} if invoked by application
        *          logic.
        */
       static void run(Database database, boolean bootstrap)
       {
          SchemaCheck check = instances.computeIfAbsent(database, SchemaCheck::new);

          synchronized (check)
          {
             check.processChanges(bootstrap);
          }
       }

       /**
        * Private constructor, meant to be called only from {@link #run(Database, boolean)}.
        *
        * @param   database
        *          Database whose schema is being processed.
        */
       private SchemaCheck(Database database)
       {
          this.database = database;
          this.dmoPackage = DMO_BASE_PACKAGE + '.' + database.getName();
       }

       /**
        * Notify the FWD runtime that a mutable database schema has been modified. This causes the specified
        * database's schema to be scanned and compared with a previously captured state, for the purpose of
        * updating the FWD runtime's state to match the detected schema changes.
        * <p>
        * This is intended to be a development-time operation and should not be invoked while the target
        * database is in active use, as DMO classes will be added and potentially removed as part of the update.
        */
       public void processChanges(boolean bootstrap)
       {
          try
          {
             SchemaComparator comparator = SchemaComparator.getInstanceExistingState(database);
             ComparisonResult result = bootstrap
                                       ? comparator.getFreshResult()
                                       : comparator.compareWithPrevious();
             if (!result.isSame())
             {
                SchemaValidator.validate(result.getState().getSequences());

                comparator.enrichExtentFields(result);

                Set<Table> tablesToUnload = new HashSet<>();
                Set<Table> tablesToLoad = new HashSet<>();
                for (Difference difference : result.getDifferences())
                {
                   // Sequences are not attached to tables directly. Differences in sequences (or any other change
                   // not related to tables) should not reload any classes.
                   if (difference.getUpdatedTable() == null)
                   {
                      continue;
                   }

                   Element newElement = difference.getNewElement();
                   if (newElement != null)
                   {
                      // New element added, we will be loading a new DMO
                      tablesToLoad.add(difference.getUpdatedTable());

                      if (!(newElement instanceof Table))
                      {
                         // It's not a table that was created.
                         // Table already existed, we will be unloading the old version.
                         tablesToUnload.add(difference.getUpdatedTable());
                      }
                   }

                   Element oldElement = difference.getOldElement();
                   if (oldElement != null)
                   {
                      // Table was dropped, we will remove the class
                      tablesToUnload.add(difference.getUpdatedTable());

                      if (!(oldElement instanceof Table))
                      {
                         // It's not a table that was dropped. Table still exists, we will re-create the class.
                         tablesToLoad.add(difference.getUpdatedTable());
                      }
                   }
                }

                // deregister obsolete tables
                // TODO: this alone will not allow DMO classes to unload; need to sever all strong references in
                // the runtime, including in the ProxyFactory cache. For now, this is best efforts (just release
                // the classloader's references), and we load a new class instead of re-using the old name.
                tablesToUnload.forEach(this::deregisterTable);

                // Register tables
                for (Table table : tablesToLoad)
                {
                   try
                   {
                      registerTable(table);
                   }
                   catch (Exception ex)
                   {
                      // Table validation failed, removing it from result so that it can be reloaded next time.
                      comparator.removeTableFromLastKnownState(table);
                      throw ex;
                   }
                }
             }
          }
          catch (PersistenceException exc)
          {
             LOG.log(Level.SEVERE, "State of database " + database + " could not be read/written.", exc);
             throw new StopConditionException("Failed to update mutable database '" + database + "'.", exc);
          }
       }

       /**
        * Table was dropped. This method will remove the DMO that was previously registered for the table.
        * <p>
        * TODO: this will remove the strong reference within the {@code AsmClassLoader}, but we also need to
        * remove strong references to the DMO interface (and implementation class) throughout the runtime.
        *
        * @param   table
        *          Table object
        */
       private void deregisterTable(Table table)
       {
          DmoMeta dmoMeta = knownTables.remove(table.getName());
          if (dmoMeta == null)
          {
             LOG.warning("Imbalanced table registration. Cannot deregister " + table.getName() +
                         " because it was not registered, or was already deregistered.");
             return;
          }

          String className = dmoMeta.getImplementationClass().getName();
          String ifaceName = dmoMeta.getDmoImplInterface().getName();

          try
          {
             AsmClassLoader asmLoder = AsmClassLoader.getInstance();
             asmLoder.unloadClass(className);
             asmLoder.unloadClass(ifaceName);
             asmLoder.unloadClass(ifaceName + "$Buf");
          }
          catch (ClassNotFoundException e)
          {
             LOG.log(Level.SEVERE, "Class " + className + " could not be unloaded.", e);
          }

          TableMapper.removeDynamicTable(dmoMeta);
          DmoMetadataManager.deregisterDmos(ifaceName, className);
          MetadataManager.removeDynamicTable(database, dmoMeta);
       }

       /**
        * Table was created. This method will create a DMO for the table.
        *
        * @param   table
        *          A table object which contains information on the DMO/table to be constructed and register.
        */
       private void registerTable(Table table)
       {
          SchemaValidator.validate(table);

          long fieldId = 1;
          Map<String, String> clsFields4GLtoORM = new HashMap<>();
          for (Attribute field : table.getAttributes())
          {
             if (field.isPrimaryKey())
             {
                // skip surrogate primary key
                continue;
             }

             String fieldName4GL = field.getName();
             String fieldName4GLNorm = TextOps.rightTrimLower(fieldName4GL);
             String fieldNameORM = ReservedProperty.isReservedProperty(fieldName4GL)
                                   ? fieldName4GL
                                   : "field" + (fieldId++);
             clsFields4GLtoORM.put(fieldName4GLNorm, fieldNameORM);
          }

          DmoMeta dmoMeta = null;
          String ifaceName = getInterfaceName(table);
          try
          {
             String tableName = table.getName();
             Map<String, String> legacyAttributes = processComment(table.getComment());
             String legacyTableName = getTokenValue(legacyAttributes, LEGACY_NAME, tableName);
             String dumpName =        getTokenValue(legacyAttributes, DUMP_NAME,   table.getName());
             // TODO: add other tokens

             Aast root = generateSchemaAst(table, ifaceName, tableName, legacyTableName, clsFields4GLtoORM);

             ConversionPool.Results results = ConversionPool.runTask(ConversionProfile.P2O, root);
             root = (Aast) results.getStoredObject("p2o.prog");
             ConversionPool.runTask(ConversionProfile.P2O_POST, results, root);

             root = (Aast) results.getStoredObject("p2o.peer");
             root.putAnnotation("permanent", true);
             root.putAnnotation("package", DMO_BASE_PACKAGE);

             // the dump-name is mandatory and unique when saved to _file meta table
             Aast tableNode = root.getImmediateChild(P2OLookup.CLASS, null);
             tableNode.putAnnotation("dump-name", dumpName);

             // generate interface Java ASTs
             results = ConversionPool.runTask(ConversionProfile.JAVA_DMO, root);
             Aast iface = (Aast) results.getStoredObject(DMO_IFACE_CATEGORY, ifaceName);
             String fullIfaceName = dmoPackage + "." + ifaceName;

             // generate bytecode from Java ASTs
             results = ConversionPool.runTask(ConversionProfile.BREW_DMO_ASM, iface);

             AsmClassLoader loader = AsmClassLoader.getInstance();
             Map<String, Class<?>> classes = new HashMap<>();
             byte[] code;

             // load interfaces first
             List<String> keys = (List<String>) results.getStoredObject("iface.keys");
             for (String name : keys)
             {
                try
                {
                   code = (byte[]) results.getStoredObject("ifaces", name);
                   Class<?> clazz = loader.loadDynamicClass(name, code);
                   classes.put(name, clazz);
                }
                catch (ClassNotFoundException e)
                {
                   LOG.log(Level.SEVERE, "Class " + name + " could not be loaded.", e);
                   throw e; // will be caught below and wrapped in a StopConditionException
                }
             }

             Class<? extends DataModelObject> dmoIface =
                (Class<? extends DataModelObject>) classes.get(fullIfaceName);
             dmoMeta = DmoMetadataManager.registerDmo(dmoIface);

             Class<? extends Record> dmoClass = dmoMeta.getImplementationClass();
             String fullImplClassName = dmoClass.getName();
             classes.put(fullImplClassName, dmoClass);

             if (Persistence.isActive())
             {
                MetadataManager.addDynamicTableToFile(table.getDatabase(), dmoIface.getName());
             }

             knownTables.put(tableName, dmoMeta);

             LOG.info("Successfully registered new DMO " + fullIfaceName);
          }
          catch (Exception e)
          {
             if (dmoMeta != null)
             {
                // rollback. Make sure no meta information remains behind
                TableMapper.removeDynamicTable(dmoMeta);
                DmoMetadataManager.deregisterDmos(ifaceName, dmoMeta.getImplementationClass().getName());
                MetadataManager.removeDynamicTable(table.getDatabase(), dmoMeta);
             }
             throw new StopConditionException("DMO creation and registration failed", e);
          }
          finally
          {
             // clean up after TRPL rules in the AST registry
             AstManager.get().removeTree(getAstPath(ifaceName));
          }
       }

       /**
        * Get the path representing the DMO interface in the AST registry.
        *
        * @param   ifaceName
        *          Short name of DMO interface.
        *
        * @return  AST registry path name for the given DMO interface.
        */
       private String getAstPath(String ifaceName)
       {
          return DMO_BASE_PACKAGE.replace('.', '/') + "/" + database.getName() + "/" + ifaceName + ".java";
       }

       /**
        * Convert table name to interface name for DMOs.
        *
        * @param table Table object
        */
       private String getInterfaceName(Table table)
       {
          long tableId = tableCounter.getAndIncrement();

          return table.getName() + "_" + tableId;
       }

       /**
        * Generate AST that represents the target table and has the same structure as .p2o files from
        * the conversion stage.
        *
        * @param  table
        *         Object containing definitions of the target table.
        * @param  ifaceName
        *         The name of the target DMO interface.
        * @param  tableName
        *         The name of the target SQL table.
        * @param  legacyTableName
        *         The legacy (Progress) name of the table.
        * @param  fields4GLtoORM
        *         Map 4GL field names to ORM names.
        *
        * @return the root node of the generated AST.
        */
       Aast generateSchemaAst(Table table,
                              String ifaceName,
                              String tableName,
                              String legacyTableName,
                              Map<String, String> fields4GLtoORM)
       {
          Dialect dialect = Dialect.getDialect(database);
          Aast root = new ProgressAst(new CommonToken(ProgressParserTokenTypes.DATABASE, database.getName()));

          ProgressAst tableAst = new ProgressAst(new CommonToken(ProgressParserTokenTypes.TABLE, tableName));
          tableAst.putAnnotation("class-name", ifaceName);
          tableAst.putAnnotation("table-name", tableName);
          tableAst.putAnnotation(P2OLookup.HISTORICAL, legacyTableName);
          root.addChild(tableAst);

          Set<Attribute> attributesToSkip = getAttributesToSkip(table.getAttributes(), dialect);

          // add field nodes to table
          Map<String, Aast> fieldAsts = new HashMap<>();
          for (Attribute field : table.getAttributes())
          {
             if (attributesToSkip.contains(field))
             {
                // skip surrogate primary key, timezone field for DTTZ, etc.
                continue;
             }

             String fieldName4GL = TextOps.rightTrimLower(field.getName());
             String fieldNameORM = fields4GLtoORM.get(fieldName4GL);

             Map<String, String> legacyAttributes = processComment(field.getComment());

             ParmType parmType = dialect.translateToParmType(field.getType(), legacyAttributes.get(LEGACY_TYPE));
             int legacyType = propTypeToProgress.get(parmType);

             ProgressAst fieldAst = new ProgressAst(new CommonToken(legacyType, fieldNameORM));
             fieldAsts.put(fieldName4GL, fieldAst);
             ReservedProperty resPropName = ReservedProperty.getByName(fieldName4GL);
             if (resPropName != null)
             {
                fieldAst.putAnnotation("__fieldid", (long) resPropName.id);
             }

             fieldAst.putAnnotation(P2OLookup.HISTORICAL,
                                    getTokenValue(legacyAttributes, LEGACY_NAME, field.getName()));
             if (legacyAttributes.containsKey(LABEL))
             {
                fieldAst.putAnnotation(P2OLookup.LABEL, legacyAttributes.get(LABEL));
             }
             if (legacyAttributes.containsKey(COLUMN_LABEL))
             {
                fieldAst.putAnnotation(P2OLookup.COLUMN_LABEL, legacyAttributes.get(COLUMN_LABEL));
             }
             if (legacyAttributes.containsKey(FORMAT))
             {
                fieldAst.putAnnotation(P2OLookup.FORMAT, legacyAttributes.get(FORMAT));
             }
             // TODO: add any other attributes of the fields

             Long dec = null;
             if (legacyAttributes.containsKey(DIGITS))
             {
                try
                {
                   dec = Long.parseLong(legacyAttributes.get(DIGITS));
                }
                catch (NumberFormatException e)
                {
                   LOG.log(Level.INFO, "Failed to parse DECIMALS from SQL comment: " + e.getMessage());
                }
             }
             if (dec == null)
             {
                dec = dialect.getDecimals(field.getType(), field.getPrecision(), null /*ignored*/);
             }
             if (dec != null)
             {
                fieldAst.putAnnotation(P2OLookup.DECIMALS, dec);
             }

             long extent = field.getExtent();
             if (extent > 0)
             {
                fieldAst.putAnnotation(P2OLookup.EXTENT, extent);
             }

             fieldAst.putAnnotation(P2OLookup.NON_NULLABLE, field.getNullable());

             if (parmType == ParmType.CHAR || parmType == ParmType.TEXT)
             {
                boolean cs = DialectHelper.getCaseSensitive(legacyAttributes.get(CASE_SENSITIVE));
                fieldAst.putAnnotation(P2OLookup.CASE_SENSITIVE, cs);
             }

             tableAst.addChild(fieldAst);

             // add initial value node to field

             String valStr = field.getDefaultValue();
             Integer valType = null;

             if (valStr == null          ||
                 SQL_NULL.equals(valStr) ||
                 (parmType == ParmType.CHAR && ResourceIdHelper.UNKNOWN_STRING.equals(valStr)))
             {
                valType = ProgressParserTokenTypes.UNKNOWN_VAL;
                valStr = ResourceIdHelper.UNKNOWN_STRING;
             }
             else if (!valStr.isEmpty())
             {
                switch (parmType)
                {
                   case CHAR:
                      valType = ProgressParserTokenTypes.STRING;
                      valStr = character.javaRuntimeToProgressSourceString(valStr);
                      break;
                   case INT64:
                   case INT:
                   case RECID:
                      valType = ProgressParserTokenTypes.NUM_LITERAL;
                      break;
                   case DEC:
                      valType = ProgressParserTokenTypes.DEC_LITERAL;
                      break;
                   case LOG:
                      valType = dialect.resolveBooleanConstant(valStr)
                                ? ProgressParserTokenTypes.BOOL_TRUE
                                : ProgressParserTokenTypes.BOOL_FALSE;
                      break;
                   case DATE:
                      valType = ProgressParserTokenTypes.DATE_LITERAL;
                      break;
                   case DT:
                      valType = ProgressParserTokenTypes.DATETIME_LITERAL;
                      break;
                   case DTTZ:
                      valType = ProgressParserTokenTypes.DATETIME_TZ_LITERAL;
                      break;
                   case RAW:
                      valType = null;
                      break;
                   default:
                      valType = null;
                      if (LOG.isLoggable(Level.WARNING))
                      {
                         LOG.warning("Unsupported initial value literal type in mutable table field: " +
                                     parmType + "; (" + legacyTableName + "." + field.getName() + ")");
                      }
                }
             }

             if (valType != null)
             {
                ProgressAst initNode = new ProgressAst(new CommonToken(ProgressParserTokenTypes.KW_INIT, ""));
                fieldAst.addChild(initNode);
                ProgressAst initVal = new ProgressAst(new CommonToken(valType, valStr));
                initNode.addChild(initVal);
             }
          }

          // add index nodes to table
          for (Index index : table.getIndexes())
          {
             if (index.getPrimary())
             {
                // skip surrogate primary key
                continue;
             }

             String indexName = TextOps.rightTrimLower(index.getName());
             ProgressAst indexAst = new ProgressAst(new CommonToken(ProgressParserTokenTypes.INDEX, indexName));
             tableAst.addChild(indexAst);

             if (index.getUnique())
             {
                indexAst.putAnnotation(P2OLookup.UNIQUE, true);
             }

    //         if (index.isWord())
    //         {
    //            indexAst.putAnnotation(P2OLookup.WORD, true);
    //         }

             // add index fields to index
             for (Tuple<String, Boolean> col: index.getColumns())
             {
                String col4GL = TextOps.rightTrimLower(col.getKey());
                ProgressAst comp = new ProgressAst(new CommonToken(ProgressParserTokenTypes.INDEX_FIELD, col4GL));
                indexAst.addChild(comp);
                Boolean sortDir = col.getValue();
                boolean isAscending = sortDir == null || sortDir;
                comp.addChild(new ProgressAst(isAscending
                            ? new CommonToken(ProgressParserTokenTypes.KW_ASCEND, "ASCENDING")
                            : new CommonToken(ProgressParserTokenTypes.KW_DESCEND, "DESCENDING")));
                if (!isAscending)
                {
                   comp.putAnnotation("descend", true);
                }
             }
          }

          assignIds(root, 1);

          // now assign each index field's refid to the corresponding field's ID
          Aast indexAst = tableAst.getImmediateChild(ProgressParserTokenTypes.INDEX, null);
          while (indexAst != null)
          {
             Aast compAst = indexAst.getImmediateChild(ProgressParserTokenTypes.INDEX_FIELD, null);
             while (compAst != null)
             {
                Aast fieldAst = fieldAsts.get(dialect.extractColumnName(compAst.getText()));
                if (fieldAst != null)
                {
                   compAst.putAnnotation("refid", fieldAst.getId());
                   compAst = indexAst.getImmediateChild(ProgressParserTokenTypes.INDEX_FIELD, compAst);
                }
                else
                {
                   // 'recid' column is normal to occur as the last component of a non-unique index
                   if (!Session.PK.equals(compAst.getText()))
                   {
                      LOG.severe("Failed to process " + compAst.getText() + " index component.");
                      throw new StopConditionException(
                            "Failed to process " + compAst.getText() + " index component on mutable table.");
                   }

                   compAst = null;
                }
             }

             indexAst = tableAst.getImmediateChild(ProgressParserTokenTypes.INDEX, indexAst);
          }

          return root;
       }

       /**
        * Get a list of attributes that should not be handled as a DMO property. This is to provide unified
        * checks for surrogate primary key columns, timezone for DTTZ attributes, etc.
        *
        * @param   attributes
        *          All available columns.
        * @param   dialect
        *          Database dialect for datatype translation purposes.
        *
        * @return  Set of attributes that should not be handled as a DMO property.
        */
       private Set<Attribute> getAttributesToSkip(Collection<Attribute> attributes, Dialect dialect)
       {
          return attributes.stream()
             .map(field -> {
                if (field.isPrimaryKey())
                {
                   // skip surrogate primary key
                   return field;
                }

                Map<String, String> legacyAttributes = processComment(field.getComment());
                ParmType parmType = dialect.translateToParmType(field.getType(), legacyAttributes.get(LEGACY_TYPE));
                if (parmType == ParmType.DTTZ)
                {
                   // find field that has the same name + _offset, skip this field later.
                   Optional<Attribute> timezoneColumn = attributes.stream()
                      .filter(a -> a.getName().equalsIgnoreCase(field.getName() + DDLGeneratorWorker.DTZ_OFFSET))
                      .findFirst();
                   if (timezoneColumn.isPresent())
                   {
                      return timezoneColumn.get();
                   }
                }
                return null;
             })
             .filter(Objects::nonNull)
             .collect(Collectors.toSet());
       }

       /**
        * Process a comment and split in pairs of (token, value). The pairs are separated by comma, the tokens
        * always end with a colon.
        *
        * @param   comment
        *          The comment as read from SQL remark.
        *
        * @return  A map of pairs token/value.
        */
       public static Map<String, String> processComment(String comment)
       {
          Map<String, String>  ret = new HashMap<>();
          if (comment == null)
          {
             return ret;
          }

          String[] splits = comment.split(DELIMITERS);
          for (String split : splits)
          {
             int i = split.indexOf(':'); // separate token from value using colon
             if (i > 0)
             {
                String token = split.substring(0, i + 1).trim();
                if (!COMMENT_TOKENS.contains(token))
                {
                   LOG.log(Level.INFO, "Unknown token type: '" + token + "'.");
                }
                ret.putIfAbsent(token, split.substring(i + 1).trim());
             }
             else
             {
                LOG.log(Level.INFO, "Failed to parse Token: Value ('" + split + "')");
             }
          }
          return ret;
       }

       /**
        * Returns an attribute from the map or default value otherwise.
        *
        * @param   attributes
        *          The map of legacy attributes.
        * @param   token
        *          The token
        * @param   def
        *          the default value to be returned in case the token is not found.
        *
        * @return  The value of the attribute or the default specified if token not found.
        */
       public static String getTokenValue(Map<String, String> attributes, String token, String def)
       {
          String ret = attributes.get(token);
          return ret == null ? def : ret;
       }
    }