DynamicConversionHelper.java

/*
** Module   : DynamicConversionHelper.java
** Abstract : Base class with common functions for DynamicQueryHelper and DynamicValidationHelper.
**
** Copyright (c) 2017-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
** 001 SVL 20171024 Created initial version.
** 002 CA  20190717 Added COM-HANDLE, BLOB, CLOB, RAW, ROWID types.
**     CA  20190722 Added object type.
**     CA  20190826 force_dmo_alias must be set only at the buffer used in the query, and not at
**                  the default buffer, when there is an explicit buffer used by the query.
** 003 OM  20200906 New ORM implementation.
** 004 OM  20200919 Improved access to dmo metadata.
**     OM  20200924 P2JIndexComponent carries multiple information to avoid map lookups for them.
**     CA  20200924 Performance improvement for prepareTempTable - cache the buffer DMO and schema ASTs, and
**                  also re-use the schema dictionary scope.
**     OM  20201001 Improved DMO manipulation performance by caching slow Property annotation access.
**     OM  20201208 Fixed model and schema AST caching for dynamic DMOs.
**     OM  20210127 Replaced TableMapper.getLegacyName() triple map lookup with direct access to local
**                  dmoMeta.legacyTable.
**     OM  20210309 Do not use DmoMeta as key in TableMapper because temp-tables share the same DmoMeta.
**     ECF 20210506 Use RecordBuffer.getDmoInfo() getter instead of direct field access to prevent
**                  NPE in proxy case.
**     ECF 20210511 Replaced DynamicTablesHelper.normalizeName with TextOps.rightTrimLower.
**     CA  20210709 Added support for rowid and recid injected variables.
**     ECF 20210914 Minor format cleanup.
**     ME  20211229 Allow backslash escape character in strings only for UNIX.
**     ECF 20220103 Minor format cleanup.
**     OM  20220224 Fixed preprocessing of literals extracted from a converted query.
**     OM  20220522 Added "force_dmo_alias" annotation in all cases to be sure the buffer names match.
**     TJD 20220504 Upgrade do Java 11 minor changes
**     SVL 20230110 P2JIndex.components() provides direct access to the components array in order to improve
**                  performance.
**     OM  20230112 Small optimizations: cache local values, replaced double map lookup from TableMapper with
**                  direct access.
**     SVL 20230113 Improved performance by replacing some "for-each" loops with indexed "for" loops.
**     CA  20230116 Avoid using handle.unwrap, handle.getReference or other BDT usage from within FWD runtime.
** 005 ECF 20230301 Fixed prepareTempTable regression introduced by dynamic, persistent DMO buffer. May need
**                  further refactoring.
** 006 OM  20230215 Handled collision of buffers with same dynamic table name.
** 007 AD  20231211 Modified injectVariable() method to handle conversion error cases as 4GL does,
**                  using injectVariableHelper() method and InjectVariableContext inner class.
** 008 OM  20240318 Improved support for dynamic tables in mutable databases.
** 009 CA  20240410 Compiler constants need to be processed as variables - added support for buffer state 
**                  constants (ROW-UNMODIFIED/CREATED/MODIFIED/DELETED).
** 010 OM  20240410 The information on mutable databases are mergeFromAst() into global scope.
** 011 AL2 20240530 Prepare the dictFile as well to be able to work with SchemaWorker in run-time conversion.
** 012 AI  20240605 Updated processString to corectly process ' and ".
** 013 AS  20240905 Fixed 223 error condition handling.
** 014 ICP 20250129 Used int64.of and logical.of to leverage caches instances.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
**
**   0. Attribution Requirement.
**
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
**
**   1. No License To Use Trademarks.
**
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
**
**   2. No Misrepresentation of Affiliation.
**
**     You may not represent yourself as Golden Code Development Corporation or FWD.
**
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
**
**   3. No Misrepresentation of Source or Origin.
**
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.persist;

import antlr.*;
import com.goldencode.ast.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.persist.orm.*;
import com.goldencode.p2j.schema.*;
import com.goldencode.p2j.uast.*;
import com.goldencode.p2j.util.*;
import java.util.*;
import java.util.Map;

/**
 * Base class with common functions for {@link DynamicQueryHelper} and
 * {@link DynamicValidationHelper}.
 */
abstract class DynamicConversionHelper
{
   /** Map of P2J wrappers to their associated token type. */
   private static final Map<String, Integer> wrapperToTokenType = new HashMap<>();
   
   static
   {
      wrapperToTokenType.put(ParmType.CHAR.toString(),      ProgressParserTokenTypes.FIELD_CHAR);
      wrapperToTokenType.put(ParmType.INT.toString(),       ProgressParserTokenTypes.FIELD_INT);
      wrapperToTokenType.put(ParmType.INT64.toString(),     ProgressParserTokenTypes.FIELD_INT64);
      wrapperToTokenType.put(ParmType.DEC.toString(),       ProgressParserTokenTypes.FIELD_DEC);
      wrapperToTokenType.put(ParmType.LOG.toString(),       ProgressParserTokenTypes.FIELD_LOGICAL);
      wrapperToTokenType.put(ParmType.DATE.toString(),      ProgressParserTokenTypes.FIELD_DATE);
      wrapperToTokenType.put(ParmType.DT.toString(),        ProgressParserTokenTypes.FIELD_DATETIME);
      wrapperToTokenType.put(ParmType.DTTZ.toString(),      ProgressParserTokenTypes.FIELD_DATETIME_TZ);
      wrapperToTokenType.put(ParmType.HANDLE.toString(),    ProgressParserTokenTypes.FIELD_HANDLE);
      wrapperToTokenType.put(ParmType.OBJECT.toString(),    ProgressParserTokenTypes.FIELD_CLASS);
      wrapperToTokenType.put(ParmType.COMHANDLE.toString(), ProgressParserTokenTypes.FIELD_COM_HANDLE);
      wrapperToTokenType.put(ParmType.RECID.toString(),     ProgressParserTokenTypes.FIELD_RECID);
      wrapperToTokenType.put(ParmType.ROWID.toString(),     ProgressParserTokenTypes.FIELD_ROWID);
      wrapperToTokenType.put(ParmType.BLOB.toString(),      ProgressParserTokenTypes.FIELD_BLOB);
      wrapperToTokenType.put(ParmType.CLOB.toString(),      ProgressParserTokenTypes.FIELD_CLOB);
      wrapperToTokenType.put(ParmType.RAW.toString(),       ProgressParserTokenTypes.FIELD_RAW);
   }

   /**
    * Create a {@link DataModelAst} with the given token type and text.
    *
    * @param    type
    *           The node's token type.
    * @param    text
    *           The node's text.
    * @param    p2oFile
    *           The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the
    *           temp-table objects.
    *
    * @return   A {@link DataModelAst} instance setup as noted above.
    */
   private static DataModelAst createDataModelAst(int type, String text, String p2oFile)
   {
      AstManager astManager = AstManager.get();
      long treeId = astManager.getTreeId(p2oFile);
      DataModelAst ast = new DataModelAst(new CommonToken(type, text.intern()));
      ast.setId(astManager.getNextNodeId(treeId));
      
      return ast;
   }
   
   /**
    * Build an {@link DataModelAst index component} associated with the given property AST.
    *
    * @param    comp
    *           The index component
    * @param    propAst
    *           The associated {@link DataModelAst property AST}.
    * @param    p2oFile
    *           The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the
    *           temp-table objects.
    *
    * @return   See above.
    */
   private static DataModelAst buildIndexComponent(P2JIndexComponent comp,
                                                   DataModelAst propAst,
                                                   String p2oFile)
   {
      String name        = comp.getLegacyName();
      String historical  = (String) propAst.getAnnotation(P2OLookup.HISTORICAL);
      String datatype    = (String) propAst.getAnnotation(P2OLookup.DATATYPE);
      
      DataModelAst root = createDataModelAst(DataModelTokenTypes.INDEX_COL, name.intern(), p2oFile);
      root.putAnnotation(P2OLookup.HISTORICAL, historical.intern());
      root.putAnnotation(P2OLookup.DATATYPE, datatype.intern());
      if (comp.isIgnoreCase())
      {
         root.putAnnotation(P2OLookup.IGNORE_CASE, true);
      }
      if (comp.isDescending())
      {
         root.putAnnotation(P2OLookup.DESCEND, true);
      }
      if (comp.isAbbreviated())
      {
         root.putAnnotation(P2OLookup.ABBREVIATED, true);
      }
      root.putAnnotation(P2OLookup.REFID, propAst.getId());

      return root;
   }

   /**
    * Obtain the other operand of a binary operator node. This method works for binary and unary
    * operators since the next sibling is {@code null}.
    *
    * @param   progAst
    *          The node to search for sibling.
    *
    * @return  The sibling of the passed in node. If such node does not exist, {@code null} is
    *          returned. In case of ternary operator, the result is irrelevant.
    */
   private static ProgressAst getSibling(ProgressAst progAst)
   {
      ProgressAst first = (ProgressAst) progAst.getParent().getFirstChild();
      if (first == progAst)
      {
         // I am the first child, the second child if exists, is my next sibling
         return (ProgressAst) progAst.getNextSibling();
      }
      else
      {
         // there is at least one sibling before me, return that
         return first;
      }
   }

   /**
    * Build the root {@link ProgressAst} node for the schema dictionary AST.
    *
    * @param    schema
    *           The schema name.
    *
    * @return   See above.
    */
   private static ProgressAst buildRootSchema(String schema)
   {
      ProgressAst root = createProgressAst(SchemaParserTokenTypes.DATABASE, schema);
      root.putAnnotation("srcfile", "in-memory");

      return root;
   }

   /**
    * Build the {@link DataModelAst root} AST and associated with the given schema.
    *
    * @param    schema
    *           The schema name.
    * @param    p2oFile
    *           The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the
    *           temp-table objects.
    *
    * @return   See above.
    */
   private static DataModelAst buildRootDataModel(String schema, String p2oFile)
   {
      String pkg = Configuration.getParameter("pkgroot") + ".dmo";

      // this is a reverse generation of what schema/p2o.xml does
      DataModelAst root = createDataModelAst(DataModelTokenTypes.DATA_MODEL, schema, p2oFile);
      root.putAnnotation(P2OLookup.PACKAGE, pkg);

      return root;
   }

   /**
    * Create a {@link DataModelAst data model object} AST for the specified buffer.
    *
    * @param    buf
    *           The buffer instance.
    * @param    rbuf
    *           The {@link RecordBuffer} instance for the buffer.
    * @param    p2oFile
    *           The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the
    *           temp-table objects.
    *
    * @return   A {@link DataModelAst} buffer definition.
    *
    * @throws   PersistenceException
    *           In case of problems during preparation of the temp-table schema and p2o ASTs.
    */
   private static DataModelAst buildDataModelClass(Buffer buf, RecordBuffer rbuf, String p2oFile)
   throws PersistenceException
   {
      String historical = rbuf.getDmoInfo().legacyTable; // trimmed and interned
      String dmoName = rbuf.getDMOName().intern();
      
      DataModelAst root = createDataModelAst(DataModelTokenTypes.CLASS, dmoName, p2oFile);
      root.putAnnotation(P2OLookup.HISTORICAL, historical);
      
      // build the ID property. Normally, in conversion we use the name configured in p2j.cfg.xml
      // accessed via Configuration.getSchemaConfig().getPrimaryKeyName(), but since we have access
      // to runtime directory, the latter will prevail
      DataModelAst idProp = createDataModelAst(DataModelTokenTypes.PRIMARY,
                                               DatabaseManager.PRIMARY_KEY,
                                               p2oFile);
      idProp.putAnnotation(P2OLookup.DATATYPE, "Long");
      root.addChild(idProp);
      
      Map<String, DataModelAst> propMap = new HashMap<>();
      
      // go through all non-extent properties
      Class<? extends DataModelObject> dmoIface = rbuf.getDMOInterface();
      DmoMeta dmoInfo = DmoMetadataManager.getDmoInfo(dmoIface);
      for (Iterator<Property> it = dmoInfo.getFields(true); it.hasNext(); )
      {
         Property fieldAnn = it.next();
         Class<?> fieldType = fieldAnn._fwdType;
         if (!BaseDataType.class.isAssignableFrom(fieldType))
         {
            continue;
         }
         
         DataModelAst propAst = buildProperty(fieldAnn, p2oFile);
         
         root.addChild(propAst);
         
         String histName = (String) propAst.getAnnotation(P2OLookup.HISTORICAL);
         propMap.put(TextOps.rightTrimLower(histName), propAst);
      }
      
//      // add the extent properties
//      Map<String, Integer> extentMap = rbuf.getExtentMap();
//      Class<?>[] innerClasses = dmoClass.getDeclaredClasses();
//      for (Class<?> inner : innerClasses)
//      {
//         int extent = 0;
//         
//         fields = inner.getDeclaredFields();
//         for (Field field : fields)
//         {
//            if (!BaseDataType.class.isAssignableFrom(field.getType()))
//            {
//               continue;
//            }
//            
//            if (extent == 0)
//            {
//               String fname = field.getName();
//               extent = extentMap.get(fname);
//            }
//            
//            root.addChild(buildProperty(buf, field, extent, p2oFile));
//         }
//      }
      
      // composite info is not needed by P2OLookup
      Iterator<P2JIndex> idxIter = rbuf.getDmoInfo().getDatabaseIndexes(); 
      while (idxIter.hasNext())
      {
         P2JIndex index = idxIter.next();
         String idx = DBUtils.extractIndexName(index.getName());
         
         if (index.getLegacyName() != null)
         {
            // include only legacy indexes
            root.addChild(buildIndex(buf, rbuf, index, idx, propMap, p2oFile));
         }
      }
      
      // DataModelTokenTypes.UNIQUE, ONE_TO_ONE and ONE_TO_MANY nodes are not needed
      
      return root;
   }
   
   /**
    * Create a {@link DataModelAst property} AST for the specified DMO java field.
    *
    * @param    fieldAnn
    *           The annotation for the field defining this property at the DMO.
    * @param    p2oFile
    *           The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the
    *           temp-table objects.
    *
    * @return   A {@link DataModelAst} property definition.
    */
   private static DataModelAst buildProperty(Property fieldAnn, String p2oFile)
   {
      String property = fieldAnn.name;
      String historical = fieldAnn.legacy;
      
      DataModelAst root = createDataModelAst(DataModelTokenTypes.PROPERTY, property, p2oFile);
      root.putAnnotation(P2OLookup.HISTORICAL, historical);
      root.putAnnotation(P2OLookup.DATATYPE, fieldAnn._fwdType.getSimpleName().intern());
      
      if (fieldAnn.extent != 0)
      {
         root.putAnnotation(P2OLookup.EXTENT, Long.valueOf(fieldAnn.extent));
         root.putAnnotation("original", fieldAnn.original);
         root.putAnnotation("index", Long.valueOf(fieldAnn.index));
      }
      
      if (fieldAnn._isCharacter && fieldAnn.caseSensitive)
      {
         root.putAnnotation(P2OLookup.CASE_SENSITIVE, true);
      }
      
      // INITIAL is not needed for P2OLookup
      
      return root;
   }
   
   /**
    * Build a {@link DataModelAst index} for the specified buffer.
    *
    * @param    buf
    *           The {@link Buffer} instance to which the index belongs.
    * @param    rbuf
    *           The {@link RecordBuffer} instance.
    * @param    index
    *           The index definition.
    * @param    idxName
    *           The index name.
    * @param    propMap
    *           A map with the {@link DataModelAst property} ASTs by their legacy name.
    * @param    p2oFile
    *           The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the
    *           temp-table objects.
    *
    * @return   A {@link DataModelAst index} definition.
    */
   private static DataModelAst buildIndex(Buffer                    buf,
                                          RecordBuffer              rbuf,
                                          P2JIndex                  index,
                                          String                    idxName,
                                          Map<String, DataModelAst> propMap,
                                          String                    p2oFile)
   {
      DataModelAst root = createDataModelAst(DataModelTokenTypes.INDEX, idxName, p2oFile);
      
      root.putAnnotation(P2OLookup.HISTORICAL, index.getLegacyName().intern());
      if (index.isUnique())
      {
         root.putAnnotation(P2OLookup.UNIQUE, true);
      }
      if (index.isPrimary())
      {
         root.putAnnotation(P2OLookup.PRIMARY, true);
      }
      if (index.isWord())
      {
         root.putAnnotation(P2OLookup.WORD, true);
      }
      
      ArrayList<P2JIndexComponent> comps = index.components();
      for (int i = 0; i < comps.size(); i++)
      {
         P2JIndexComponent comp = comps.get(i);
         root.addChild(buildIndexComponent(comp, propMap.get(comp.getNormalizedName()), p2oFile));
      }
      
      return root;
   }
   
   /**
    * Build a temp-table definition to be loaded in the {@link SchemaDictionary}. Use the given
    * {@link DataModelAst data model object} AST as the source.
    *
    * @param    p2o
    *           The data model object AST.
    * @param    rbuf
    *           The record buffer instance.
    *
    * @return   See above.
    */
   private static ProgressAst buildTempTable(DataModelAst p2o, RecordBuffer rbuf)
   {
      DmoMeta dmoInfo = rbuf.getDmoInfo();
      String legacyTable = (String) p2o.getAnnotation("historical");
      
      ProgressAst root = createProgressAst(dmoInfo.tempTable ? SchemaParserTokenTypes.TEMP_TABLE
                                                             : SchemaParserTokenTypes.TABLE,
                                           legacyTable);
      
      int fieldIdx = 0;
      for (int i = 0; i < p2o.getNumImmediateChildren(); i++)
      {
         DataModelAst child = (DataModelAst) p2o.getChildAt(i);
         Property property = dmoInfo.getFieldInfo(child.getText());
         
         if (child.getType() == DataModelTokenTypes.PROPERTY)
         {
            String datatype = (String) child.getAnnotation("datatype");
            int ftype = wrapperToTokenType.get(datatype);
            String legacyField = (String) child.getAnnotation("historical"); // dmoProperty.legacy
            
            ProgressAst field = createProgressAst(ftype, legacyField);
            
            // the following properties are not needed by SchemaDictionary:
            // 1. sqlWidth
            // 2. extent
            // 3. decimals
            // 4. length
            // 5. maxwidth
            
            if (property.caseSensitive)
            {
               ProgressAst pcase = createProgressAst(SchemaParserTokenTypes.KW_CASE_SEN, "case-sensitive");
               pcase.addChild(createProgressAst(SchemaParserTokenTypes.BOOL_TRUE, "true"));
               field.addChild(pcase);
            }

            // add ORDER node
            ProgressAst pord = createProgressAst(SchemaParserTokenTypes.ORDER, "ORDER");
            pord.addChild(createProgressAst(SchemaParserTokenTypes.NUM_LITERAL, Long.toString(fieldIdx)));
            fieldIdx = fieldIdx + 10;

            field.addChild(pord);

            Long extent = (Long) child.getAnnotation("extent");
            if (extent != null)
            {
               ProgressAst pext = createProgressAst(SchemaParserTokenTypes.KW_EXTENT, "extent");
               pext.addChild(createProgressAst(SchemaParserTokenTypes.NUM_LITERAL, extent.toString()));

               field.addChild(pext);
            }

            root.addChild(field);
         }
         else if (child.getType() == DataModelTokenTypes.INDEX)
         {
            // add a new index
            String legacyIndex = (String) child.getAnnotation("historical");
            ProgressAst rootIdx = createProgressAst(SchemaParserTokenTypes.INDEX, legacyIndex);

            // build index properties
            ProgressAst props = createProgressAst(SchemaParserTokenTypes.PROPERTIES, "");
            if (child.isAnnotation(P2OLookup.UNIQUE))
            {
               props.addChild(createProgressAst(SchemaParserTokenTypes.KW_UNIQUE, "UNIQUE"));
            }
            if (child.isAnnotation(P2OLookup.PRIMARY))
            {
               props.addChild(createProgressAst(SchemaParserTokenTypes.KW_PRIMARY, "PRIMARY"));
            }
            if (child.isAnnotation(P2OLookup.WORD))
            {
               props.addChild(createProgressAst(SchemaParserTokenTypes.KW_WORD_IDX, "WORD"));
            }
            
            rootIdx.addChild(props);
            
            // build index components
            DataModelAst idxCol = (DataModelAst) child.getImmediateChild(DataModelTokenTypes.INDEX_COL, null);
            while (idxCol != null)
            {
               String legacyField = (String) idxCol.getAnnotation("historical");
               
               ProgressAst idxField = createProgressAst(SchemaParserTokenTypes.INDEX_FIELD, legacyField);
               if (idxCol.isAnnotation(P2OLookup.ABBREVIATED))
               {
                  idxField.addChild(createProgressAst(SchemaParserTokenTypes.KW_ABBV, "ABBREVIATED"));
               }
               
               if (idxCol.isAnnotation(P2OLookup.DESCEND))
               {
                  idxField.addChild(createProgressAst(SchemaParserTokenTypes.KW_DESCEND, "DESCENDING"));
               }
               else
               {
                  idxField.addChild(createProgressAst(SchemaParserTokenTypes.KW_ASCEND, "ASCENDING"));
               }
               
               rootIdx.addChild(idxField);
               
               idxCol = (DataModelAst) child.getImmediateChild(DataModelTokenTypes.INDEX_COL, idxCol);
            }
            
            root.addChild(rootIdx);
         }
      }

      return root;
   }

   /**
    * Create a {@link ProgressAst} with the given token type and text.
    *
    * @param    type
    *           The node's token type.
    * @param    text
    *           The node's text.
    *
    * @return   A {@link ProgressAst} instance setup as noted above.
    */
   private static ProgressAst createProgressAst(int type, String text)
   {
      return new ProgressAst(new CommonToken(type, text));
   }

   /**
    * Injects a new variable in place of a literal or field reference. This method makes the
    * following changes in the structure of the {@code progAst} tree:
    * <ul>
    *    <li>created the DEFINE statement structure/subtree of a new {@code varName} variable of
    *          the type required</li>
    *    <li>adds all necessary annotations to the newly created nodes as the
    *          {@code post-parse-fixup.xml} was run against it.</li>
    *    <li>replaces the literal or field reference with variable reference (morph it by changing
    *          its text/type and) the required annotations (including refid to newly created
    *          DEFINE VAR stmt)</li>
    *    <li>infers the type of the variable and converts the old text of the literal to a
    *          {@link BaseDataType} that will be returned. This will be used to initialize the
    *          variable just created.</li>
    * </ul>
    *
    * @param   litAst
    *          The AST of a literal or field references to be replaced by a variable.
    * @param   varName
    *          The name of the variable to be created.
    * @param   progAst
    *          The root of the procedure. It is only needed to inject the definition of the new
    *          variable.
    *
    * @return  The value as {@link BaseDataType} of the literal replaced by the new variable. If
    *          a field reference is replaced, then unknown value of proper {@link BaseDataType} is
    *          returned.
    */
   static Object injectVariable(ProgressAst litAst, String varName, ProgressAst progAst)
   {
      return injectVariable(new ProgressAst[]{litAst}, varName, progAst);
   }

   /**
    * Injects a new variable in place of a literal or field reference. This method makes the
    * following changes in the structure of the {@code progAst} tree:
    * <ul>
    *    <li>created the DEFINE statement structure/subtree of a new {@code varName} variable of
    *          the type required</li>
    *    <li>adds all necessary annotations to the newly created nodes as the
    *          {@code post-parse-fixup.xml} was run against it.</li>
    *    <li>replaces the literal or field reference with variable reference (morph it by changing
    *          its text/type and) the required annotations (including refid to newly created
    *          DEFINE VAR stmt)</li>
    *    <li>infers the type of the variable and converts the old text of the literal to a
    *          {@link BaseDataType} that will be returned. This will be used to initialize the
    *          variable just created.</li>
    * </ul>
    *
    * @param   litAsts
    *          The list of ASTs of literals or field references to be replaced by a variable. All
    *          ASTs are replaced with the same variable.
    * @param   varName
    *          The name of the variable to be created.
    * @param   progAst
    *          The root of the procedure. It is only needed to inject the definition of the new
    *          variable.
    *
    * @return  The value as {@link BaseDataType} of the literal replaced by the new variable. If
    *          a field reference is replaced, then unknown value of proper {@link BaseDataType} is
    *          returned.
    */
   static Object injectVariable(ProgressAst[] litAsts, String varName, ProgressAst progAst)
   {
      BaseDataType ret = null;
      String className = null;
      
      ProgressAst stmt = new ProgressAst();
      stmt.setType(ProgressParserTokenTypes.STATEMENT);
      stmt.setText("statement");
      progAst.graftAt(stmt, 0);  // insert in front, before buffer definitions
      
      ProgressAst define = new ProgressAst();
      define.setType(ProgressParserTokenTypes.DEFINE_VARIABLE);
      define.setText("DEFINE");
      define.putAnnotation("name", varName);
      define.putAnnotation("vardef", true);
      stmt.graft(define);
      
      ProgressAst symbol = new ProgressAst();
      symbol.setType(ProgressParserTokenTypes.SYMBOL);
      symbol.setText(varName);
      define.graft(symbol);
      
      ProgressAst as = new ProgressAst();
      as.setType(ProgressParserTokenTypes.KW_AS);
      as.setText("AS");
      define.graft(as);
      
      // step 1: evaluate the current literal as a BDT with type identical to original literal
      ProgressAst litAst = litAsts[0];
      String text = litAst.getText();
      int literalType = litAst.getType();
      switch (literalType)
      {
         // TODO: all literal types from common-progress.rules:compiler_constants need to be treated here
         
         case ProgressParserTokenTypes.KW_ROW_UMOD:
            literalType = ProgressParserTokenTypes.NUM_LITERAL;
            ret = int64.of((long)BufferImpl.ROW_UNMODIFIED);
            break;
         case ProgressParserTokenTypes.KW_ROW_DELD:
            literalType = ProgressParserTokenTypes.NUM_LITERAL;
            ret = int64.of((long)BufferImpl.ROW_DELETED);
            break;
         case ProgressParserTokenTypes.KW_ROW_MODD:
            literalType = ProgressParserTokenTypes.NUM_LITERAL;
            ret = int64.of((long)BufferImpl.ROW_MODIFIED);
            break;
         case ProgressParserTokenTypes.KW_ROW_CRTD:
            literalType = ProgressParserTokenTypes.NUM_LITERAL;
            ret = int64.of((long)BufferImpl.ROW_CREATED);
            break;
            
         case ProgressParserTokenTypes.STRING:
            ret = new character(processString(text, EnvironmentOps.isUnderWindowsFamily()));
            break;
         case ProgressParserTokenTypes.NUM_LITERAL:
            ret = new int64(text);
            break;
         case ProgressParserTokenTypes.DATE_LITERAL:
            ret = new date(text);
            break;
         case ProgressParserTokenTypes.DATETIME_LITERAL:
            ret = datetime.fromLiteral(text);
            break;
         case ProgressParserTokenTypes.DATETIME_TZ_LITERAL:
            ret = datetimetz.fromLiteral(text);
            break;
         case ProgressParserTokenTypes.DEC_LITERAL:
            ret = decimal.fromLiteral(text);
            break;
         case ProgressParserTokenTypes.HEX_LITERAL:
            literalType = ProgressParserTokenTypes.DEC_LITERAL;
            ret = decimal.fromUnsignedHexLiteral(text);
            break;
         case ProgressParserTokenTypes.BOOL_TRUE:
         case ProgressParserTokenTypes.BOOL_FALSE:
            ret = logical.of(literalType == ProgressParserTokenTypes.BOOL_TRUE);
            break;
         case ProgressParserTokenTypes.UNKNOWN_VAL: /* disabled */
            ret = new unknown();
            break;
         default:
            // is field reference
      }
      
      // step 2. create the TYPE node of a type required by the sibling. If needed, an implicit
      //         conversion is applied to value obtained in step 1. If sibling does not exist or
      //         both operands are of the same type, the conversion is skipped and the type of
      //         the original literal is used.
      int var4glType = literalType;
      switch (litAst.getParent().getType())
      {
         case ProgressParserTokenTypes.KW_EQ:
         case ProgressParserTokenTypes.EQUALS:
         case ProgressParserTokenTypes.KW_NE:
         case ProgressParserTokenTypes.NOT_EQ:
         case ProgressParserTokenTypes.KW_LT:
         case ProgressParserTokenTypes.LT:
         case ProgressParserTokenTypes.KW_LTE:
         case ProgressParserTokenTypes.LTE:
         case ProgressParserTokenTypes.KW_GT:
         case ProgressParserTokenTypes.GT:
         case ProgressParserTokenTypes.KW_GTE:
         case ProgressParserTokenTypes.GTE:
            ProgressAst sibling = getSibling(litAst);
            if (sibling != null)
            {
               if (sibling.getType() > ProgressParserTokenTypes.BEGIN_FIELDTYPES &&
                   sibling.getType() < ProgressParserTokenTypes.END_FIELDTYPES)
               {
                  // only fields are taken into consideration to avoid database CASTing issues
                  var4glType = sibling.getType();
               }
               else if (sibling.getType() == ProgressParserTokenTypes.FUNC_RECID)
               {
                  // we want our operand to be bigint to match the record.id field in db
                  var4glType = ProgressParserTokenTypes.NUM_LITERAL;
               }
            }
            break;
      }
      
      ProgressAst type = new ProgressAst();
      int varType = 0;
      InjectVariableContext methodArg = new InjectVariableContext();
      methodArg.className = className;
      methodArg.var4glType = var4glType;
      methodArg.literalType = literalType;
      methodArg.baseDataTypeObj = ret;
      methodArg.varType = varType;
      methodArg.type = type;
      methodArg.text = text;
      
      if (literalType == ProgressParserTokenTypes.STRING)
      {
         final BaseDataType[] retTempOutput = new BaseDataType[1];
         
         ErrorManager.silent(() -> 
         {
            // assign unknown to the result initially, in case conversion fails
            retTempOutput[0] = new character();
            // try to convert argument to the type of the field, but catch any thrown exceptions
            retTempOutput[0] = injectVariableHelper(methodArg);
         });
         ret = retTempOutput[0];
      }
      else 
      {
         ret = injectVariableHelper(methodArg);
      }  
      
      varType = methodArg.varType;
      type = methodArg.type;
      text = methodArg.text;
      className = methodArg.className;
      
      as.graft(type);
      
      // morphing the literal node to variable
      for (ProgressAst ast : litAsts)
      {
         ast.setText(varName);
         ast.setType(varType);
         ast.removeAnnotation("is-literal");    // remove old annotation
         ast.putAnnotation("oldtype", Long.valueOf(ProgressParserTokenTypes.SYMBOL));
         ast.putAnnotation("refid", define.getId());
      }
      
      // annotate the variable definition with correct type
      define.putAnnotation("type", Long.valueOf(varType));
      define.putAnnotation("javaname", varName);
      define.putAnnotation("classname", className);
      
      return ret;
   }
   
   /**
    * Helper method for {@link #injectVariable(ProgressAst[], String, ProgressAst)}.
    * This method could sometimes throw an exception depending on the validity
    * of the input data, but that error should sometimes be caught in the caller
    * method.
    * 
    * @param   context 
    *          An object containing all the data necessary for returning the result.
    *          
    * @return  The conversion result from the given object in the 
    *          {@link InjectVariableContext} to the type of the field, also 
    *          specified in the given argument.
    */
   private static BaseDataType injectVariableHelper(InjectVariableContext context)
   {
      BaseDataType ret = context.baseDataTypeObj;
      ProgressAst type = context.type;
      String className = context.className;
      String text = context.text;
      int var4glType = context.var4glType;
      int literalType = context.literalType;
      int varType = context.varType;
      
      try
      {
         switch (var4glType)
         {
            case ProgressParserTokenTypes.STRING:
            case ProgressParserTokenTypes.FIELD_CHAR:
               type.setType(ProgressParserTokenTypes.KW_CHAR);
               type.setText("CHARACTER");
               className = "character";
               varType = ProgressParserTokenTypes.VAR_CHAR;
               if (literalType != ProgressParserTokenTypes.STRING)
               {
                  ErrorManager.recordOrShowError(223, true, false);
               }
               break;
            case ProgressParserTokenTypes.NUM_LITERAL:
            case ProgressParserTokenTypes.FIELD_INT:
            case ProgressParserTokenTypes.FIELD_INT64:
               type.setType(ProgressParserTokenTypes.KW_INT64);
               type.setText("INT64");
               className = "int64";
               varType = ProgressParserTokenTypes.VAR_INT64;
               if (literalType != ProgressParserTokenTypes.NUM_LITERAL)
               {
                  if (literalType == ProgressParserTokenTypes.BOOL_TRUE ||
                      literalType == ProgressParserTokenTypes.BOOL_FALSE ||
                      literalType == ProgressParserTokenTypes.DATETIME_TZ_LITERAL ||
                      literalType == ProgressParserTokenTypes.DATETIME_LITERAL ||
                      literalType == ProgressParserTokenTypes.DATE_LITERAL)
                  {
                     ErrorManager.recordOrShowError(223, true, false);
                  }
                  //bad conversion when ret is of type rowid
                  ret = new int64(ret);
               }
               break;
            case ProgressParserTokenTypes.DATE_LITERAL:
            case ProgressParserTokenTypes.FIELD_DATE:
               type.setType(ProgressParserTokenTypes.KW_DATE);
               type.setText("DATE");
               className = "date";
               varType = ProgressParserTokenTypes.VAR_DATE;
               if (literalType != ProgressParserTokenTypes.DATE_LITERAL)
               {
                  if (literalType != ProgressParserTokenTypes.STRING)
                  {
                     ErrorManager.recordOrShowError(223, true, false);
                  }
                  
                  BaseDataType[] retTempOutput = new BaseDataType[1];
                  BaseDataType retTemp = ret;
                  ErrorManager.silent(() -> 
                  {
                     // assign unknown to the result initially, in case conversion fails
                     retTempOutput[0] = new date();
                     // try to convert argument to the type of the field, 
                     // but catch any thrown exceptions
                     retTempOutput[0] = new date(retTemp);
                  });
                  ret = retTempOutput[0];
               }    
               break;
            case ProgressParserTokenTypes.DATETIME_LITERAL:
            case ProgressParserTokenTypes.FIELD_DATETIME:
               type.setType(ProgressParserTokenTypes.KW_DATETIME);
               type.setText("DATETIME");
               className = "datetime";
               varType = ProgressParserTokenTypes.VAR_DATETIME;
               if (literalType != ProgressParserTokenTypes.DATETIME_LITERAL)
               {
                  if (literalType != ProgressParserTokenTypes.STRING)
                  {
                     ErrorManager.recordOrShowError(223, true, false);
                  }
                  ret = new datetime(ret);
               }
               break;
            case ProgressParserTokenTypes.DATETIME_TZ_LITERAL:
            case ProgressParserTokenTypes.FIELD_DATETIME_TZ:
               type.setType(ProgressParserTokenTypes.KW_DATE_TZ);
               type.setText("DATETIME-TZ");
               className = "datetimetz";
               varType = ProgressParserTokenTypes.VAR_DATETIME_TZ;
               if (literalType != ProgressParserTokenTypes.DATETIME_TZ_LITERAL)
               {
                  if (literalType != ProgressParserTokenTypes.STRING)
                  {
                     ErrorManager.recordOrShowError(223, true, false);
                  }
                  ret = new datetimetz(ret);
               }
               break;
            case ProgressParserTokenTypes.DEC_LITERAL:
            case ProgressParserTokenTypes.FIELD_DEC:
               type.setType(ProgressParserTokenTypes.KW_DEC);
               type.setText("DECIMAL");
               className = "decimal";
               varType = ProgressParserTokenTypes.VAR_DEC;
               if (literalType != ProgressParserTokenTypes.DEC_LITERAL)
               {
                  if (literalType == ProgressParserTokenTypes.BOOL_TRUE ||
                      literalType == ProgressParserTokenTypes.BOOL_FALSE ||
                      literalType == ProgressParserTokenTypes.DATETIME_LITERAL ||
                      literalType == ProgressParserTokenTypes.DATETIME_TZ_LITERAL ||
                      literalType == ProgressParserTokenTypes.DATE_LITERAL)  
                  {
                     ErrorManager.recordOrShowError(223, true, false);
                  }
                  //still wrong in case ret is of type rowid
                  ret = new decimal(ret);
               }
               break;
            case ProgressParserTokenTypes.BOOL_TRUE:
            case ProgressParserTokenTypes.BOOL_FALSE:
            case ProgressParserTokenTypes.FIELD_LOGICAL:
               type.setType(ProgressParserTokenTypes.KW_LOGICAL);
               type.setText("LOGICAL");
               className = "logical";
               varType = ProgressParserTokenTypes.VAR_LOGICAL;
               if (literalType != ProgressParserTokenTypes.BOOL_TRUE &&
                   literalType != ProgressParserTokenTypes.BOOL_FALSE)
               {
                  if (literalType != ProgressParserTokenTypes.STRING)
                  {
                     ErrorManager.recordOrShowError(223, true, false);
                  }  
                  BaseDataType[] retTempOutput = new BaseDataType[1];
                  BaseDataType retTemp = ret;
                  ErrorManager.silent(() -> 
                  {
                     // assign unknown to the result initially, in case conversion fails
                     retTempOutput[0] = new logical();
                     // try to convert argument to the type of the field, 
                     // but catch any thrown exceptions
                     retTempOutput[0] = new logical(retTemp);
                  });
                  ret = retTempOutput[0];                
               }
               break;
            case ProgressParserTokenTypes.FIELD_RECID:
               type.setType(ProgressParserTokenTypes.KW_RECID);
               type.setText("RECID");
               className = "recid";
               varType = ProgressParserTokenTypes.VAR_RECID;
               if (literalType == ProgressParserTokenTypes.BOOL_TRUE ||
                   literalType == ProgressParserTokenTypes.BOOL_FALSE ||
                   literalType == ProgressParserTokenTypes.DATETIME_TZ_LITERAL ||
                   literalType == ProgressParserTokenTypes.DATETIME_LITERAL ||
                   literalType == ProgressParserTokenTypes.DATE_LITERAL || 
                   literalType == ProgressParserTokenTypes.DEC_LITERAL)
               {
                  ErrorManager.recordOrShowError(223, true, false);
               }
               BaseDataType[] retTempOutput = new BaseDataType[1];
               ErrorManager.silent(() -> 
               {
                  // assign unknown to the result initially, in case conversion fails
                  retTempOutput[0] = new recid();
                  // try to convert argument to the type of the field, 
                  // but catch any thrown exceptions
                  // still improper conversion when text holds a rowid value.
                  retTempOutput[0] = new recid(text);
               });
               ret = retTempOutput[0];  
               break;
            case ProgressParserTokenTypes.FIELD_ROWID:
               type.setType(ProgressParserTokenTypes.KW_ROWID);
               type.setText("ROWID");
               className = "rowid";
               varType = ProgressParserTokenTypes.VAR_ROWID;
               if (literalType != ProgressParserTokenTypes.STRING)
               {
                  ErrorManager.recordOrShowError(223, true, false);
               }
               ret = new rowid(text);
               break;
            default:
               // throw exception?
         }
      }
      finally
      {
         context.className = className;
         context.literalType = literalType;
         context.varType = varType;
         context.baseDataTypeObj = ret;
         context.type = type;
         context.var4glType = var4glType;
         context.text = text;
      }
      
      return ret;
   }
   
   /**
    * Preprocess a literal extracted from a converted query. The String literals encountered in a AST tree
    * were already (partially) processed by the 4GL lexer. This method adjusts the string literals back to
    * what was the expected value as the resul to be identical to 4GL.
    * 
    * @param   text
    *          The text to be processed.
    * @param   windows
    *          Emulate windows. If {@code true} the {@code \} is used as a normal character, file separator.
    *          Otherwise, honour the Linux convention, where the {@code \} is an escape character with same
    *          role as {@code ~} character.
    *
    * @return  The processed text.
    */
   private static String processString(String text, boolean windows)
   {
      // test if starts with ' or "
      char ch = text.charAt(0);
      if (ch != '"' && ch != '\'')
      {
         throw new IllegalArgumentException("Missing opening quote (" + ch + ").");
      }
      
      int lastCharPos = text.lastIndexOf(ch);
      if (lastCharPos <= 0)
      {
         throw new IllegalArgumentException("Missing closing quote (" + ch + ").");
      }
      lastCharPos--;
      
      StringBuilder sb = new StringBuilder();
      for (int k = 1; k <= lastCharPos; k++)
      {
         ch = text.charAt(k);
         char next = k < lastCharPos ? text.charAt(k + 1) : '\0';
         switch (ch)
         {
            case 10:
               // the chr(10) ('\n') seems to be silently dropped in dynamic queries
               continue;
               
            case '\\':
               if (windows)
               {
                  break; 
               }
               // else: Linux: '\' is used with same value as '~', break through...
               
            case '~':
               switch (next)
               {
                  case '"':  sb.append('"');       k++; break;
                  case '\'': sb.append('\'');      k++; break;
                  case '~':  sb.append('~');       k++; break;
                  case '{':  sb.append('{');       k++; break;
                  case 't':  sb.append('\t');      k++; break; // tab
                  case 'r':  sb.append('\r');      k++; break; // CR
                  case 'n':  sb.append('\n');      k++; break; // LF
                  case 'E':  sb.append((char) 27); k++; break; // escape
                  case 'b':  sb.append('\b');      k++; break; // bell
                  case 'f':  sb.append('\f');      k++; break; // form feed
                  case 'u':
                     if (k + 4 < lastCharPos)
                     {
                        char hexVal = getEncodedChar(text, k + 2, 4, 16);
                        if (hexVal != 0xFFFF)
                        {
                           sb.append(hexVal);
                           k += 5; // 'u' + 4 hex digits
                        }
                     }
                     break;
                  case 'U':
                     if (k + 6 < lastCharPos)
                     {
                        char hexVal = getEncodedChar(text, k + 2, 6, 16);
                        if (hexVal != 0xFFFF)
                        {
                           sb.append(hexVal);
                           k += 7; // 'U' + 6 hex digits
                        }
                     }
                     break;
                  default:
                     if (next >= '0' && next <= '9' && k + 3 < lastCharPos)
                     {
                        char octVal = getEncodedChar(text, k + 1, 3, 8);
                        if (octVal != 0xFFFF)
                        {
                           sb.append(octVal);
                           k += 3; // 3 oct digits
                        }
                     }
                     break;
               }
               continue; // do not append '~'
            case '"':
               sb.append(ch);
               k = text.charAt(0) == '"' ? k + 1 : k;
               continue;
            case '\'':
               sb.append(ch);
               k = text.charAt(0) == '\'' ? k + 1 : k;
               continue;
         }
         
         sb.append(ch);
      }
      
      return sb.toString();
   }
   
   /**
    * Extract an encoded character on a specified number of digits ina specific base.
    * 
    * @param   text
    *          The text which contains the encoded character.
    * @param   k
    *          The offset where the encoding begins.
    * @param   n
    *          The number of digits used.
    * @param   base
    *          The base to be used when parsing the character.
    *
    * @return  The decoded character or {@code 0xFFFF} on error.
    */
   private static char getEncodedChar(String text, int k, int n, int base)
   {
      try
      {
         return (char) Integer.parseInt(text.substring(k, k + n), base);
      }
      catch (Exception e)
      {
         return 0xFFFF; 
      }
   }
   
   /**
    * Process the definitions of the temp-tables (associated with the temporary buffers in the
    * given list) and those from mutable databases to create the associated dictionary and p2o ASTs.
    * <p>
    * Also, it appends to the progress code string any needed 4GL code (like DEFINE BUFFER
    * statements for explicit buffers, to refer back the originating table.
    *
    * @param   dict
    *          The schema dictionary where the table definitions will be loaded.
    * @param   pcode
    *          Append any new progress code here.
    * @param   p2oFile
    *          The name of the {@link DataModelAst} tree with the temp-table objects for the query or
    *          validation statement.
    * @param   dictFile
    *          The name of the {@link ProgressAst} tree with the temp-table objects for conversion
    *          {@link SchemaDictionary}
    * @param   buffers
    *          The buffers used by the query or validation statement.
    * @param   tempTableNames
    *          The mapping of the new table names (to avoid temp-table name conflicts).
    *
    * @throws  PersistenceException
    *          In case of problems during preparation of the temp-table schema and p2o ASTs.
    */
   static void prepareDynamicTables(SchemaDictionary dict,
                                    StringBuilder pcode,
                                    String p2oFile,
                                    String dictFile,
                                    ArrayList<Buffer> buffers,
                                    Map<Buffer, String> tempTableNames)
   throws PersistenceException
   {
      Map<String, Object[]> astRoots = new HashMap<>();
      
      long tableOrder = 0;
      for (int i = 0; i < buffers.size(); i++)
      {
         BufferImpl impl = (BufferImpl) buffers.get(i);
         RecordBuffer rbuf = impl.buffer();
         rbuf.initialize();
         
         String legacyBufName = impl.buffer().getLegacyName();
         String legacyTableName = rbuf.getDmoInfo().legacyTable;
         
         boolean renameTable = tempTableNames != null && rbuf.getDmoInfo().tempTable;
         if (renameTable)
         {
            legacyTableName += "_tt_" + rbuf.getDmoInfo().sqlTable; // this should make it unique
            tempTableNames.put(impl, legacyTableName);
         }
         
         if (rbuf.isTemporary() || rbuf.isMutable())
         {
            String dbName = rbuf.isTemporary() ? DatabaseManager.TEMP_TABLE_SCHEMA
                                               : impl.getDbName().toJavaType();
            Object[] rootObjs = astRoots.get(dbName);
            DataModelAst p2oRoot;
            ProgressAst schemaRoot;
            StringBuilder sb;
            if (rootObjs == null)
            {
               p2oRoot = buildRootDataModel(dbName, p2oFile);
               p2oRoot.brainwash(p2oFile);
               
               schemaRoot = buildRootSchema(dbName);
               sb = new StringBuilder();
               
               if (rbuf.isMutable())
               {
                  // mark ASTs as mutable database
                  p2oRoot.putAnnotation(SchemaDictionary.MUTABLE_ANNOTATION, true);
                  schemaRoot.putAnnotation(SchemaDictionary.MUTABLE_ANNOTATION, true);
               }
               
               astRoots.put(dbName, new Object[] {p2oRoot, schemaRoot, sb});
            }
            else
            {
               p2oRoot =    (DataModelAst)  rootObjs[0];
               schemaRoot = (ProgressAst)   rootObjs[1];
               sb =         (StringBuilder) rootObjs[2];
            }
            
            sb.append(rbuf.getDMOAlias())
              .append('-')
              .append(rbuf.getDMOImplementationClass().getName())
              .append(';');
            
            // create the schema and p2o definition for this temp-table
            DataModelAst model = rbuf.getDataModelAst();
            if (model == null)
            {
               model = buildDataModelClass(impl, rbuf, p2oFile);
               rbuf.setDataModelAst(model);
            }
            else
            {
               model.setParent(null);
               model.setNextSibling(null);
            }
            if (renameTable)
            {
               model.putAnnotation(P2OLookup.HISTORICAL, legacyTableName);
            }
            
            ProgressAst table = rbuf.getSchemaTableAst();
            if (table == null)
            {
               table = buildTempTable(model, rbuf);
               rbuf.setSchemaTableAst(table);
            }
            else
            {
               table.setParent(null);
               table.setNextSibling(null);
            }
            
            table.putAnnotation("lastOrder", tableOrder);
            tableOrder = tableOrder + 10;
            
            schemaRoot.graft(table);
            
            p2oRoot.graft(model);
         }
         
         // must create explicit buffers, if the legacy buffer name is not the same as the legacy
         // default buffer name (which is the same as the legacy table name).
         if (!legacyBufName.equalsIgnoreCase(legacyTableName))
         {
            pcode.append("DEFINE BUFFER ").append(legacyBufName).append(" FOR ");
            if (rbuf.isTemporary())
            {
               pcode.append("TEMP-TABLE ");
            }
            pcode.append(legacyTableName).append(".\n");
         }
      }
      
      // save the P2O tree, to expose it to the conversion rules
      for (Map.Entry<String, Object[]> entry : astRoots.entrySet())
      {
         AstManager.get().saveTree((DataModelAst) entry.getValue()[0], p2oFile, true);
         
         ProgressAst schemaRoot = (ProgressAst) entry.getValue()[1];
         AstManager.get().saveTree(schemaRoot, dictFile, true);
         
         if (schemaRoot.isAnnotation(SchemaDictionary.MUTABLE_ANNOTATION))
         {
            // merge into existing schema (it may be partially statically defined)
            dict.mergeFromAst(schemaRoot);
         }
         else
         {
            // load the temp schema
            String key = entry.getValue()[2].toString();
            if (!dict.reuseCachedScope(key))
            {
               dict.loadFromAst(schemaRoot, false, true);
               dict.cacheScope(key);
            }
         }
      }
   }
   
   /**
    * Prepare the query AST by placing the appropriate annotations with the converted java names
    * for the external program and also by created bogus
    * {@link ProgressParserTokenTypes#DEFINE_TEMP_TABLE} nodes (which would be created from a
    * {@code DEFINE TEMP-TABLE tt.} statement, with no fields); the latter is needed to allow
    * proper conversion of the default buffer name.
    *
    * @param   root
    *          The root AST of the converted progress code.
    * @param   buffers
    *          The buffers used by the query or validation statement.
    * @param   fileName
    *          The file name of the class containing the converted code.
    * @param   javaClassName
    *          The name of the generated java class.
    * @param   localTableName
    *          A map with local table names, in case of a name conflict.
    */
   static void prepareTree(AnnotatedAst root,
                           ArrayList<Buffer> buffers,
                           String fileName,
                           String javaClassName,
                           Map<Buffer, String> localTableName)
   {
      // Create a bogus DEFINE TEMP-TABLE for each temp buffer (just to let their converted
      // buffer names resolve properly).
      // Dynamic buffers have unique DMO aliases, so "some-buffer" name may be converted to the
      // alias like "someTable__1", while aliases in predicate will be converted using standard
      // P2J rules, which will result to the converted alias "someTable". So below we explicitly
      // specify converted DMO alias like "someTable__1" for the conversion rules using
      // "force_dmo_alias" annotation.
      for (int i = 0; i < buffers.size(); i++)
      {
         BufferImpl impl = (BufferImpl) buffers.get(i);
         RecordBuffer rbuf = impl.buffer();
         String legacyTableName = localTableName == null ? rbuf.getDmoInfo().legacyTable.toLowerCase() :
                                  localTableName.getOrDefault(impl, rbuf.getDmoInfo().legacyTable.toLowerCase());
         if (!rbuf.isTemporary())
         {
            continue;
         }
         
         ProgressAst stmt = createProgressAst(ProgressParserTokenTypes.STATEMENT, "statement");
         stmt.putAnnotation("reachable", true);
         
         ProgressAst defChild = createProgressAst(ProgressParserTokenTypes.DEFINE_TEMP_TABLE, "DEFINE");
         defChild.putAnnotation("reachable", true);
         defChild.putAnnotation("schemaname", legacyTableName);
         defChild.putAnnotation("bufname", legacyTableName);
         defChild.putAnnotation("dbname", "");
         
         // the dynamic conversion might come up with a different buffer name than the one from the caller.
         // Force the name of the buffer name to be the same as in statically converted code.
         defChild.putAnnotation("force_dmo_alias", rbuf.getDMOAlias());
         
         if (rbuf.isDynamic())
         {
            if (DynamicTablesHelper.isDynamicTableBuffer(rbuf))
            {
               defChild.putAnnotation("is_dynamic_table", true);
            }
         }
         stmt.graft(defChild);
         
         ProgressAst ttChild = createProgressAst(ProgressParserTokenTypes.KW_TEMP_TAB, "TEMP-TABLE");
         ttChild.putAnnotation("reachable", true);
         defChild.graft(ttChild);
         
         ProgressAst symChild = createProgressAst(ProgressParserTokenTypes.SYMBOL, legacyTableName);
         symChild.putAnnotation("reachable", true);
         ttChild.graft(symChild);
         
         root.graftAt(stmt, 0);
      }
      
      // put annotation related to name, package, etc
      root.putAnnotation("file", fileName);
      root.putAnnotation("classname", javaClassName);
      root.putAnnotation("javaname", javaClassName);
      root.putAnnotation("relative-name", fileName);
      root.putAnnotation("basename", fileName);
      root.putAnnotation("package-base", "");
      root.putAnnotation("pkgname", "com.goldencode.p2j.persist.dynquery");
      
      // make "force_dmo_alias" annotations, errors are ignored at this point, they will
      // be raised later anyway
      Iterator<Aast> iter = root.iterator();
      while (iter.hasNext())
      {
         Aast chAst = iter.next();
         
         if (chAst.getType() == ProgressParserTokenTypes.RECORD_PHRASE ||
             chAst.getType() == ProgressParserTokenTypes.DEFINE_BUFFER)
         {
            Aast bufAst = (Aast) chAst.getFirstChild();
            String bufName = (String) bufAst.getAnnotation("bufname");
            if (bufName != null)
            {
               int dotPos = bufName.lastIndexOf(".");
               if (dotPos >= 0)
               {
                  bufName = bufName.substring(dotPos + 1);
               }
               
               int idx = locateBuffer(buffers, bufName);
               if (idx >= 0)
               {
                  BufferImpl impl = (BufferImpl) buffers.get(idx);
                  bufAst.putAnnotation("force_dmo_alias", impl.buffer().getDMOAlias());
                  if (DynamicTablesHelper.isDynamicTableBuffer(impl.buffer()))
                  {
                     bufAst.putAnnotation("is_dynamic_table", true);
                  }
               }
            }
         }
      }
   }
   
   /**
    * Locate a {@link Buffer} with the given {@code name}, from the specified list of buffers,
    * and return its position.
    *
    * @param    buffers
    *           List of buffers to be searched.
    * @param    name
    *           The buffer name.
    *
    * @return   The 0-based position of this buffer or -1 if not found.
    */
   static int locateBuffer(ArrayList<Buffer> buffers, String name)
   {
      for (int i = 0; i < buffers.size(); i++)
      {
         if (name.equalsIgnoreCase(((BufferImpl) buffers.get(i)).buffer().getLegacyName()))
         {
            return i;
         }
      }
      
      return -1;
   }
   
   /**
    * Helper inner class used to send necesary data when calling 
    * {@link #injectVariableHelper(InjectVariableContext)}, and also be able to remember the
    * changes made to the values of the class instance.
    */
   private static class InjectVariableContext
   {
      /** The object to be converted. */
      private BaseDataType baseDataTypeObj;
      
      /** Variable used to save the new data type of the object. */
      private ProgressAst type;
      
      /** Variable used to save the name of the class to which the object was converted to. */
      private String className;
      
      /** String value of the object to be converted. Used in the cases of rowid and recid. */
      private String text;
      
      /** 
       * {@link ProgressParserTokenTypes} correspondent of the data type the object need 
       *  to be converted to. 
       */
      private int var4glType;
      
      /** The current type of the object that need to be converted. */
      private int literalType;
      
      /** 
       * {@link ProgressParserTokenTypes} value that corresponds to a variable with the type
       *  the objects needs converting to.
       */
      private int varType;
   }
}