# RecordMeta.java

    /*
    ** Module   : RecordMeta.java
    ** Abstract : Information required to perform CRUD operations on a specific DMO type.
    **
    ** Copyright (c) 2019-2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
    ** 001 ECF 20191015 First revision. Information required to perform basic CRUD operations on a
    **                  specific DMO type.
    **     OM  20200202 Some fixes and support for multiple columns (datetime-tz) added.
    ** 002 ECF 20200929 Changed unique index validation SQL to allow statements to be UNION ALL'd together as a
    **                  performance optimization.
    **     OM  20201001 Improved DMO manipulation performance by caching slow Property annotation access.
    **     OM  20201002 Use DmoMeta cached information instead of map lookups.
    **     OM  20201007 Replaced inlined column name with TemporaryBuffer.MULTIPLEX_FIELD_NAME.
    **     ECF 20210204 Removed word index unsupported log message.
    **     OM  20211020 Added _DATASOURCE_ROWID property.
    **     OM  20220713 Allowed denormalized properties carrying the same fieldId (of same extent field) to be
    **                  processed by readPropertyMeta().
    **     OM  20220727 FieldId and PropertyId are different for denormalized extent fields. Both are computed
    **                  at conversion time.
    **     ECF 20221005 Added support for dynamic initial values.
    **     CA  20221114 Added a helper array to keep the index of the normalized extent properties in the DMO
    **                  'props' array.
    **     CA  20220826 A permanent table can be defined with no fields, fixed 'readPropertyMeta' to allow this.
    **     ECF 20221017 Fixed regression in dynamic initial value implementation.
    **     CA  20221114 Added a helper array to keep the index of the normalized extent properties in the DMO
    **                  'props' array.
    ** 003 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    ** 004 OM  20240215 Added getIndexKeyCollisionsSQLs() method which returns the statements needed to select the
    **                  PKs of the records which collide with current record in unique indices.
    ** 005 AP  20250129 Added getter for loadSql.
    ** 006 AS  20250220 Added initialDataNullProps.
    **                  Small performance improvement for applyInitialValues.
    ** 007 OM  20250325 Added missing javadocs.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist.orm;

    import java.lang.reflect.*;
    import java.util.*;
    import java.util.function.*;
    import java.util.logging.*;
    import com.goldencode.p2j.persist.*;
    import com.goldencode.p2j.persist.annotation.*;
    import com.goldencode.p2j.persist.dialect.*;
    import com.goldencode.p2j.util.logging.*;
    import org.apache.commons.lang3.tuple.*;

    /**
     * Metadata which is associated with the concrete subclass implementation of a database record,
     * needed to perform basic CRUD operations on records.
     */
    public final class RecordMeta
    {
       /** Logger */
       private static final CentralLogger log = CentralLogger.get(RecordMeta.class.getName());

       /**
        * Thread-local, temporary association of a {@code DataModelObject} interface to its
        * {@code RecordMeta}. They are the last {@code DataModelObject} processed and the obtained
        * {@code crtRecordMeta}. We do a double check to be sure the initialization of the static
        * field {@code _metadata} is correct (and the DMO interface is the right one).
        * This pair of values is kept until {@link #get} is called and is reset after the first valid
        * access this static method.
        *
        * @see  #get
        */
       private static final ThreadLocal<Pair<Class<? extends DataModelObject>, RecordMeta>> local =
          new ThreadLocal<>();

       /** Names of the backing database tables */
       public final String[] tables;

       /** The legacy name of the parent table. */
       public final String legacyName;

       /** Values to which a newly created record should be set */
       final Object[] initialData;

       /** Bit set which marks the position of each non-null column in the backing table */
       final BitSet nonNullableProps;

       /** Bit set which marks the position of each property participating in an index */
       final BitSet allIndexedProps;

       /** Bit set which marks the position of each property that can be validated */
       final BitSet validatable;

       /** Bit set which marks the position of each property that is null*/
       final BitSet initialDataNullProps;

       /** Bit sets which mark the positions of the component columns of unique indices */
       final BitSet[] uniqueIndices;

       /** Bit sets which mark the positions of the component columns of non-unique indices */
       final BitSet[] nonuniqueIndices;

       /** SQL query statement(s) to access data needed to load a single DMO instance */
       final String[] loadSql;

       /**
        * The index in {@link #props} array where each {@link #loadSql} statement has the properties
        * (with the same extent) stored.  The value of <code>loadSqlIndexes[0]</code> is always zero, as the
        * {@link #loadSql}'s first element is the scalar properties SELET.
        */
       final int[] loadSqlIndexes;

       /** SQL insert statement(s) which insert a single DMO's data into the database */
       final String[] insertSql;

       /** SQL delete statement(s) which delete a single DMO's data from the database */
       final String[] deleteSql;

       /** Does this object represent metadata for a temporary table? */
       private final boolean temporary;

       /**
        * Property metadata, ordered scalar properties first, then the extent properties grouped by
        * their extent size. Each group represents a series, all series are sorted by their common
        * extent size. This is the 'persistence' order.
        */
       private final PropertyMeta[] props;

       /** Array of names of unique indices, with positional correspondents in {@code uniqueIndices} list */
       private final String[] uniqueIndicesNames;

       /**
        * The array of legacy names for unique indices, with positional correspondents in
        * {@code uniqueIndices} list.
        */
       private final String[] uniqueIndicesLegacyNames;

       /**
        * The array of names for non-unique indices, with positional correspondents in {@code nonuniqueIndices}
        * list.
        */
       private final String[] nonuniqueIndicesNames;

       /**
        * The array of legacy names for non-unique indices, with positional correspondents in
        * {@code nonuniqueIndices} list.
        */
       private final String[] nonuniqueIndicesLegacyNames;

       /** The {@code DmoMeta} structure which contains the legacy meta information. */
       private final DmoMeta dmoMeta;

       /** Stores the offset of each property, mapped by their name. */
       private final Map<String, Integer> reversedIndexProperties = new HashMap<>();

       /** Map of index name to index UID (positive for unique indices, negative for non-unique indices) */
       private final Map<String, Integer> indexIdByName = new HashMap<>();

       /** UID of primary index, if any. A value of {@code 0} indicates no primary index exists */
       private final int primaryIndex;

       /** Dynamic initializers, if any. */
       private final DynamicInitializer[] dynamicInitializers;

       /** The SQL statements used to test the uniqueness of the record against each unique index. */
       private String[] uniqueSql;

       /**
        * The set of DELETE SQL statements used to remove existing records that conflict with a record ready to be
        * inserted. Only used for FILL operations in REPLACE mode.
        */
       private String[] replaceCleanupSql = null;

       /**
        * The set of SQL query statements used to extract existing records that conflict with a record ready to be
        * inserted. Only used for FILL operations in REPLACE mode.
        */
       private String[] getIndexKeyCollisionsSQLs = null;

       /** The {@code Dialect} which was used for creating the {@code uniqueSql} statements. */
       private Dialect uniqueSqlDialect = null;

       /**
        * The {@code Dialect} which was used for creating the {@code replaceCleanupSql} statements.
        * <p>
        * Normally this should be H2 as this is the only dialect supported for temp-tables, as the
        * {@code replaceCleanupSql} will be used only on _temp database.
        */
       private Dialect replaceCleanupSqlDialect = null;

       /**
        * The {@code Dialect} which was used for creating the {@code getIndexKeyCollisionsSQLs} statements.
        * <p>
        * Normally this should be H2 as this is the only dialect supported for temp-tables, as the
        * {@code getIndexKeyCollisionsSQLs} will be used only on _temp database.
        */
       private Dialect getIndexKeyCollisionsDialect = null;

       /**
        * Initialize the record's metadata. This should only be called once per concrete DMO
        * implementation subclass, when that class is loaded.
        *
        * @param   dmoIface
        *          DMO interface; the public API to business logic. Annotations on the getter
        *          methods of this interface are used to define the concrete subclass.
        * @param   dmoMeta
        *          The {@code DmoMeta} structure which contains the legacy meta information. May have not been
        *          fully filled yet.
        */
       RecordMeta(Class<? extends DataModelObject> dmoIface, DmoMeta dmoMeta)
       {
          if (dmoIface == null)
          {
             throw new NullPointerException("Parameter cannot be null.");
          }

          this.dmoMeta = dmoMeta;
          temporary = dmoMeta.isTempTable();

          legacyName = dmoMeta.legacyTable;
          props = RecordMeta.readPropertyMeta(dmoMeta);
          tables = RecordMeta.tableNames(dmoMeta, props);
          loadSql = Loader.composeLoadStatements(tables[0], props, temporary);
          loadSqlIndexes = Loader.composeLoadStatementsIndexes(props);
          insertSql = Persister.composeInsertStatements(tables[0], props, temporary);
          deleteSql = Persister.composeDeleteStatements(tables);

          nonNullableProps = new BitSet(props.length);

          // collect index data for record flushing purposes
          List<BitSet> unique = new ArrayList<>();
          List<String> uniqueNames = new ArrayList<>();
          List<String> uniqueLegacyNames = new ArrayList<>();
          List<BitSet> nonunique = new ArrayList<>();
          List<String> nonuniqueNames = new ArrayList<>();
          List<String> nonuniqueLegacyNames = new ArrayList<>();
          String[] primary = new String[] { null };
          validatable = new BitSet();
          initialDataNullProps = new BitSet();
          allIndexedProps = readIndices(dmoMeta.getAnnotatedInterface(), props, primary, validatable,
                                        unique, uniqueNames, uniqueLegacyNames,
                                        nonunique, nonuniqueNames, nonuniqueLegacyNames);
          uniqueIndices = unique.toArray(new BitSet[unique.size()]);
          uniqueIndicesNames = uniqueNames.toArray(new String[uniqueNames.size()]);
          uniqueIndicesLegacyNames = uniqueLegacyNames.toArray(new String[uniqueLegacyNames.size()]);
          nonuniqueIndices = nonunique.toArray(new BitSet[nonunique.size()]);
          nonuniqueIndicesNames = nonuniqueNames.toArray(new String[nonuniqueNames.size()]);
          nonuniqueIndicesLegacyNames = nonuniqueLegacyNames.toArray(new String[nonuniqueLegacyNames.size()]);

          // populate the map of indices by their names (indicesByName), the unique indexes have positive ids,
          // the non-unique negative, both 1-base. However, the keys in this tables are string and guaranteed
          // to be also unique
          int indexId = 1;
          for (int k = 0; k < uniqueIndices.length; k++)
          {
             indexIdByName.put(uniqueIndicesNames[k], indexId++);
          }
          indexId = -1;
          for (int k = 0; k < nonuniqueIndices.length; k++)
          {
             indexIdByName.put(nonuniqueIndicesNames[k], indexId--);
          }

          String pi = primary[0];
          primaryIndex = pi == null ? 0 : indexIdByName.get(pi);

          // collect default initial data values and initialize non-nullable properties bitset
          int len = props.length;
          initialData = new Object[len];
          Map<String, DynamicInitializer> dynInit = null;
          for (int i = 0; i < len; i++)
          {
             PropertyMeta pm = props[i];

             // collect default initial data values; except for dynamic initial values, the elements are
             // immutable and thus the references to these instances can be copied to every newly created
             // record which needs to be initialized to default data values.
             if (pm.initialValue instanceof Supplier)
             {
                Supplier<Object> supplier = (Supplier<Object>) pm.initialValue;

                if (dynInit == null)
                {
                   dynInit = new HashMap<>();
                }

                // dynamic initializers are handled specially; note that this leaves a null value in the
                // initialData array at this index
                DynamicInitializer init =
                   dynInit.computeIfAbsent(pm.getType().getSimpleName() + ':' + pm.dmoProperty.initial,
                                           (k) -> new DynamicInitializer(supplier, props.length));
                init.registerProperty(i);
             }
             else
             {
                // the simple case: static initial value, just store the immutable initial value
                initialData[i] = pm.initialValue;
                initialDataNullProps.set(i, initialData[i] == null);
             }

             if (pm.isMandatory())
             {
                nonNullableProps.set(i);
             }

             // ensure we are on the first property for an extent case (so we can build the offset from that)
             reversedIndexProperties.putIfAbsent(pm.getName(), i);
          }

          // store dynamic initializers, if any; we no longer need the initial string keys, just the initializers
          dynamicInitializers = dynInit == null
                                ? null
                                : dynInit.values().toArray(new DynamicInitializer[dynInit.size()]);

          // everything went smooth, prepare to set the data to final [_metadata] static member of the
          // new class to be build:
          local.set(Pair.of(dmoIface, this));
       }

       /**
        * Get the last created {@code RecordMeta}. After the first access the value is reset.
        * <p>
        * To avoid processing the {@link #readPropertyMeta} twice (one before creating the DMO
        * implementation class and one after, when its {@code _metadata} is statically initialized at
        * first access, we store the last created {@code RecordMeta}) we store the latest created
        * object in a static member and return it at first call. The {@code RecordMeta} is called
        * in a synchronized block so that nobody else will get/reset the result. The {@code key}
        * parameter makes sure the new metadata goes to the right class.
        *
        * @param   key
        *          The DMO interface that was previously processed. It must be the interface that the
        *          DMO implementation ask for meta data.
        *
        * @return  The last processed {@code RecordMeta} that corresponds to {@code key} DMO
        *          interface. If the method is called multiple time, {@code null} is retuned for
        *          subsequent invocation.
        */
       public static RecordMeta get(Class<? extends DataModelObject> key)
       {
          if (key == null)
          {
             return null;
          }

          Pair<Class<? extends DataModelObject>, RecordMeta> pair = local.get();
          if (pair == null)
          {
             return null;
          }

          if (pair.getKey() == key)
          {
             RecordMeta meta = pair.getValue();
             // reset the value, it must be requested only once, when the class is loaded
             local.set(null);
             return meta;
          }

          // invalid key
          return null;
       }

       /**
        * Read the getter method annotations from the DMO interface and generate property metadata
        * from them. Return the metadata as an array, ordered as described in the {@link Loader}
        * class description. This order is critical to all CRUD operations performed by the loader
        * and persister, and reflects the order of the data stored in the record's core data array.
        *
        * @param   dmoMeta
        *          DMO metadata.
        *
        * @return  Array of property metadata.
        */
       static PropertyMeta[] readPropertyMeta(DmoMeta dmoMeta)
       {
          SortedSet<PropertyMeta> set = new TreeSet<>();

          // size of the returned array will be larger than the size of the set, if there are extent
          // fields in the table, because we flatten out the array to include all elements
          int size = 0;

          Class<? extends DataModelObject> dmoIface = dmoMeta.getAnnotatedInterface();
          Class<? extends DataModelObject> baseIface = dmoMeta.getDmoImplInterface();

          boolean tempTable = Temporary.class.isAssignableFrom(dmoIface);
          if (tempTable)
          {
             // add the reserved properties, always first
             set.add(new PropertyMeta(ReservedProperty._ERRORFLAG));
             set.add(new PropertyMeta(ReservedProperty._ORIGINROWID));
             set.add(new PropertyMeta(ReservedProperty._DATASOURCEROWID));
             set.add(new PropertyMeta(ReservedProperty._ERRORSTRING));
             set.add(new PropertyMeta(ReservedProperty._PEERROWID));
             set.add(new PropertyMeta(ReservedProperty._ROWSTATE));
             size += 6;
          }

          Iterator<Property> propIt = dmoMeta.getFields(false);
          while (propIt.hasNext())
          {
             Property prop = propIt.next();
             Method meth = prop.annMethod;
             if (prop.propId < 0 || meth == null) // skip reserved properties emitted at the DMO interface
             {
                continue;
             }

             // if the extent property was denormalized, treat it as a simple field
             boolean isExtent = prop.extent != 0 && prop.index == 0;
             if (isExtent && meth.getParameterTypes().length != 1)
             {
                throw new RuntimeException(
                   "Invalid method annotated as 'Property' for an extent in DMO " + baseIface +
                   ", legacy field #" + prop.id + ": " + meth);
             }

             String setterName = meth.getName();
             if (setterName.startsWith("get"))
             {
                setterName = "set" + setterName.substring(3);
             }
             else if (setterName.startsWith("is"))
             {
                setterName = "set" + setterName.substring(2);
             }
             else
             {
                setterName = null;
             }

             Method setter = null;
             if (setterName != null)
             {
                for (Method m : dmoIface.getDeclaredMethods())
                {
                   if (m.getName().equals(setterName))
                   {
                      if (isExtent)
                      {
                         Class<?>[] types = m.getParameterTypes();
                         if (types.length != 2)
                         {
                            continue; // this is not the right one
                         }
                         if (types[0] != meth.getParameterTypes()[0])
                         {
                            continue; // still not the one we need
                         }
                      }

                      setter = m;
                      break;
                   }
                }
             }
             if (setter == null)
             {
                throw new RuntimeException(
                   "Failed to detect setter for property ID " + prop.propId + "(" + prop.name +
                   ") in DMO " + baseIface);
             }

             PropertyMeta propMeta = new PropertyMeta(prop, meth.getName(), setter.getName());

             // add to sorted set
             set.add(propMeta);

             // add 1 to size for scalar field, or extent value for extent field
             int extent = propMeta.getExtent();
             size += extent == 0 ? 1 : extent;
          }

          // convert the set contents into a flat array, with one slot per field element;
          PropertyMeta[] all = new PropertyMeta[size];
          int i = 0;
          int crtSerieStartAt = 0;
          int lastExtent = 0;
          int colIdx = 1;
          int seriesSize = 0;

          for (PropertyMeta propMeta : set)
          {
             // extent will be 0 for scalar (including denormalized) fields; > 0 for extent fields
             int extent = propMeta.getExtent();

             // set load index; this can only be done after the meta objects have been sorted:
             // for scalar fields, load index is the PropertyMeta's index in the PropertyMeta array;
             // for extent fields, it is the one-based position of the field among like-extent fields
             if (extent == 0)  // scalar value
             {
                // colIdx = i + 1; // adjust for 1-based index
                seriesSize++;
             }
             else if (extent > lastExtent) // new extent series
             {
                // set series size of first propMeta in the extent series
                if (seriesSize > 0)
                {
                   all[crtSerieStartAt].seriesSize = seriesSize;
                }

                // reset variables for the new table
                colIdx = 1;
                seriesSize = 1;
                crtSerieStartAt = i;
             }
             else // next field in same extent series
             {
                // colIdx++;
                seriesSize++;
             }
             propMeta.columnIndex = colIdx;
             propMeta.columnCount = propMeta.isDateTimeTz() ? 2 : 1;
             propMeta.offset = i;
             colIdx += propMeta.columnCount; // prepare column index for next property

             // scalar fields each get a unique PropertyMeta instance; extent field elements share a
             // single PropertyMeta instance which describes them all
             int sublen = (extent == 0) ? 1 : extent;
             for (int j = 0; j < sublen; j++, i++)
             {
                all[i] = propMeta;
             }

             lastExtent = extent;
          }

          // set series size for last series
          if (size > 0)
          {
             PropertyMeta propMeta = all[crtSerieStartAt];
             propMeta.seriesSize = seriesSize;
          }

          // debug only:
          if (log.isLoggable(Level.FINE))
          {
             log.log(Level.FINE, baseIface.getName() + " Metadata:");
             for (PropertyMeta pm : all)
             {
                log.log(Level.FINE, pm.toString());
             }
          }

          return all;
       }

       /**
        * Apply all dynamic and static initial values to a record's data array, and record which elements of the
        * array are set to {@code null} in the given bit set.
        *
        * @param   data
        *          Record's data array.
        *
        * @return   A BitSet representing the properties having null values;
        */
       BitSet applyInitialValues(Object[] data)
       {
          BitSet nullProps;
          if (dynamicInitializers == null)
          {
             nullProps = (BitSet) initialDataNullProps.clone();
             System.arraycopy(initialData, 0, data, 0, initialData.length);
          }
          else
          {
             nullProps = new BitSet();
             // if the schema defines any dynamic initializers for this record, set them.
             int len = dynamicInitializers.length;
             for (int i = 0; i < len; i++)
             {
                dynamicInitializers[i].apply(data);
             }
             // now copy static initial data into the data array; it is assumed the data array matches the length
             // of the initialData array
             len = initialData.length;
             for (int i = 0; i < len; i++)
             {
                // skip any value already set by dynamic initializers
                if (data[i] != null)
                {
                   continue;
                }
                // set each static initial datum into data array and update related state
                Object datum = initialData[i];
                data[i] = datum;
                nullProps.set(i, datum == null);
             }
          }
          return nullProps;
       }

       /**
        * Composes a set of SQL query statements that verify if a record can be safely saved to database so there
        * are no conflicts with unique indexes. Each statement selects the zero-based position of the index it is
        * meant to validate in the {@code uniqueIndices} array, as in:
        * <pre>
        *    select 0 where recid != ? and column1 = ? and column2 = ?
        * </pre>
        * for the &quot;0th&quot; index.
        * <p>
        * Multiple such statements can then be concatenated with the {@code union all} operator and the result
        * set will have a row corresponding with each unique index which failed validation, or no rows if
        * validation passed.
        *
        * @param   sqlTableName
        *          The name of the SQL table.
        * @param   d
        *          The dialect to be used when creating the queries.
        *
        * @return  An array of SELECT statements for testing a record against all unique indexes.
        */
       private String[] composeValidationSQLs(String sqlTableName, Dialect d)
       {
          int len = uniqueIndices.length;
          String[] ret = new String[len];
          for (int k = 0; k < len; k++)
          {
             StringBuilder sql = new StringBuilder();
             sql.append("select ").append(k).append(" from ").append(sqlTableName)
                .append(" where ").append(Session.PK).append(" != ?");

             if (temporary)
             {
                sql.append(" and ").append(TemporaryBuffer.MULTIPLEX_FIELD_NAME).append(" = ?");
             }

             BitSet uIndex = uniqueIndices[k];
             for (int i = uIndex.nextSetBit(0); i >= 0; i = uIndex.nextSetBit(i + 1))
             {
                PropertyMeta crtProp = props[i];
                String column = crtProp.getColumn();
                boolean csField = crtProp.isCaseSensitive();
                boolean computed = crtProp.isCharacter();

                // if this is a character field it must be tested as the index sees it: rtrimmed and
                // with proper case-sensitivity. We will make sure the parameter will also be rtrimmed and
                // converted to proper casing when set into the SQL statement
                sql.append(" and ")
                   .append(computed ? d.getProcessedCharacterColumnName(column, !csField) : column)
                   .append(" = ?");
             }

             ret[k] = sql.toString();
          }

          return ret;
       }

       /**
        * Composes a set of SQL statements that delete conflicting records from the temp-table.
        *
        * @param   sqlTableName
        *          The name of the SQL table.
        * @param   d
        *          The dialect to be used when creating the queries.
        *
        * @return  An array of SELECT statements for deleting possible conflicting records from database, one
        *          statement for each unique index.
        */
       private String[] composeReplaceCleanupSQLs(String sqlTableName, Dialect d)
       {
          int len = uniqueIndices.length;
          String[] ret = new String[len];
          for (int k = 0; k < len; k++)
          {
             StringBuilder sql = new StringBuilder();
             sql.append("delete from ").append(sqlTableName).append(" where ")
                .append(TemporaryBuffer.MULTIPLEX_FIELD_NAME).append(" = ?");

             BitSet uIndex = uniqueIndices[k];
             for (int i = uIndex.nextSetBit(0); i >= 0; i = uIndex.nextSetBit(i + 1))
             {
                PropertyMeta crtProp = props[i];
                String column = crtProp.getColumn();
                boolean csField = crtProp.isCaseSensitive();
                boolean computed = crtProp.isCharacter();

                // if this is a character field it must be tested as the index sees it: rtrimmed and
                // with proper case-sensitivity. We will make sure the parameter will also be rtrimmed and
                // converted to proper casing when set into the SQL statement
                sql.append(" and ")
                   .append(computed ? d.getProcessedCharacterColumnName(column, !csField) : column)
                   .append(" = ?");
             }

             ret[k] = sql.toString();
          }

          return ret;
       }

       /**
        * Composes a set of SQL query statements that extract conflicting records from the temp-table.
        *
        * @param   sqlTableName
        *          The name of the SQL table.
        * @param   d
        *          The dialect to be used when creating the queries.
        *
        * @return  An array of SELECT statements for deleting possible conflicting records from database, one
        *          statement for each unique index.
        */
       private String[] composeFindIndexKeyCollisionsSQL(String sqlTableName, Dialect d)
       {
          int len = uniqueIndices.length;
          String[] ret = new String[len];
          for (int k = 0; k < len; k++)
          {
             StringBuilder sql = new StringBuilder();
             sql.append("select ").append(Session.PK).append(" from ").append(sqlTableName).append(" where ")
                .append(TemporaryBuffer.MULTIPLEX_FIELD_NAME).append(" = ?");

             BitSet uIndex = uniqueIndices[k];
             for (int i = uIndex.nextSetBit(0); i >= 0; i = uIndex.nextSetBit(i + 1))
             {
                PropertyMeta crtProp = props[i];
                String column = crtProp.getColumn();
                boolean csField = crtProp.isCaseSensitive();
                boolean computed = crtProp.isCharacter();

                // if this is a character field it must be tested as the index sees it: rtrimmed and
                // with proper case-sensitivity. We will make sure the parameter will also be rtrimmed and
                // converted to proper casing when set into the SQL statement
                sql.append(" and ")
                   .append(computed ? d.getProcessedCharacterColumnName(column, !csField) : column)
                   .append(" = ?");
             }

             ret[k] = sql.toString();
          }

          return ret;
       }

       /**
        * Read the DMO interface's index annotations, if any. For each such annotation, create an
        * {@link BitSet} object which describes the components of each index. Add each such
        * object to the appropriate list of unique or non-unique index descriptors.
        * <p>
        * In addition, create and return an {@link BitSet} object which describes all indexed
        * properties of the DMO.
        *
        * @param   dmoIface
        *          DMO interface.
        * @param   props
        *          Array of metadata objects which describe each property in the DMO. The position
        *          of each element of this array will correspond with a set bit in an {@link
        *          BitSet} object which describes an index containing that property.
        * @param   primary
        *          Single-element string array output parameter, to be filled with the name of the primary
        *          index, if any.
        * @param   validatable
        *          Bit set of property positions in the record's {@code data} array of those properties which
        *          either are mandatory or which participate in a unique index.
        * @param   unique
        *          List of {@link BitSet} objects for unique indices, if any. Should be empty
        *          when this method is invoked; may be empty upon return.
        * @param   uniqueNames
        *          List of names for unique indices, if any, with positional correspondents in {@code unique}
        *          list.
        * @param   uniqueLegacyNames
        *          List of legacy names for unique indices, if any, with positional correspondents in
        *          {@code unique} list.
        * @param   nonUnique
        *          List of {@link BitSet} objects for non-unique indices, if any. Should be
        *          empty when this method is invoked; may be empty upon return.
        * @param   nonUniqueNames
        *          List of legacy names for non-unique indices, if any, with positional correspondents
        *          in {@code nonUnique} list.
        * @param   nonUniqueLegacyNames
        *          List of legacy names for non-unique indices, if any, with positional correspondents in
        *          {@code nonUnique} list.
        *
        * @return  An {@link BitSet} object which describes all properties which participate in
        *          any index of the table represented by the DMO.
        *
        * @throws  IllegalStateException
        *          if the name of an index component stored in an {@link IndexComponent} annotation
        *          does not match the name of any property in the DMO.
        */
       private static BitSet readIndices(Class<?> dmoIface,
                                         PropertyMeta[] props,
                                         String[] primary,
                                         BitSet validatable,
                                         List<BitSet> unique,
                                         List<String> uniqueNames,
                                         List<String> uniqueLegacyNames,
                                         List<BitSet> nonUnique,
                                         List<String> nonUniqueNames,
                                         List<String> nonUniqueLegacyNames)
       {
          // all bits across all indices
          BitSet allBits = new BitSet();

          Indices indices = dmoIface.getAnnotation(Indices.class);
          if (indices == null)
          {
             return allBits;
          }

          // map PropertyMeta element indices to property names
          Map<String, Integer> propMap = new HashMap<>();
          for (int i = 0; i < props.length; i++)
          {
             PropertyMeta propMeta = props[i];

             if (propMeta.getExtent() > 0)
             {
                break;
             }

             propMap.put(propMeta.getName(), i);

             if (propMeta.isMandatory())
             {
                validatable.set(i);
             }
          }

          // walk the index annotations, creating an BitSet object for each one, and updating the
          // one which describes all indexed properties
          for (Index next : indices.value())
          {
             if (next.word())
             {
                // TODO: handle word index; how do word indices affect validation?
                continue;
             }

             BitSet bits = new BitSet();
             boolean isUnique = next.unique();

             if (isUnique)
             {
                unique.add(bits);
                uniqueNames.add(next.name());
                uniqueLegacyNames.add(next.legacy());
             }
             else
             {
                nonUnique.add(bits);
                nonUniqueNames.add(next.name());
                nonUniqueLegacyNames.add(next.legacy());
             }

             if (next.primary())
             {
                primary[0] = next.name();
             }

             // walk the component annotations, determine the PropertyMeta element position which
             // matches each one, and use that to set a bit in the BitSet objects
             for (IndexComponent comp : next.components())
             {
                String field = comp.name();
                Integer j = propMap.get(field);
                if (j == null)
                {
                   throw new IllegalStateException("Invalid index field reference: '" +
                                                   field +
                                                   "' (" +
                                                   dmoIface.getName() +
                                                   "@"
                                                   + next.name() +
                                                   ")");
                }

                // set the bit corresponding with property's position in PropertyMeta array.
                bits.set(j);
                allBits.set(j);
                if (isUnique)
                {
                   validatable.set(j);
                }
             }
          }

          return allBits;
       }

       /**
        * Get the array of table names associated with this DMO. The primary table is first, followed
        * by secondary tables for extent field data, if this data is normalized. The secondary table
        * names are in ascending order by extent value.
        *
        * @param   dmoMeta
        *          The {@code DmoMeta} structure which contains the legacy meta information.
        * @param   props
        *          Array of property metadata, ordered as described in the {@link Loader} class description.
        *
        * @return  Array of table names as described above.
        */
       private static String[] tableNames(DmoMeta dmoMeta, PropertyMeta[] props)
       {
          String primary = dmoMeta.sqlTable;
          String dmoName = dmoMeta.getAnnotatedInterface().getName();
          int len = primary.length();
          if (len == 0)
          {
             throw new IllegalArgumentException("No primary table name found for DMO " + dmoName);
          }

          List<String> names = new ArrayList<>();

          // strip back-ticks, if needed
          String root = stripBackTicks(primary, dmoName);
          boolean backTicks = !root.equals(primary);

          names.add(primary);

          // scan property metadata to compose secondary table names, if needed
          int numProps = props.length;
          for (int i = 0; i < numProps; i++)
          {
             PropertyMeta pm = props[i];
             int seriesSize = pm.seriesSize;
             if (seriesSize > 0)
             {
                int extent = pm.getExtent();
                if (extent > 0)
                {
                   String name = root + "__" + extent;
                   if (backTicks)
                   {
                      name = backTickify(name);
                   }
                   names.add(name);
                   i += (seriesSize * extent);
                }
             }
          }

          return names.toArray(new String[names.size()]);
       }

       /**
        * Strip back tick quotes from a name, if it is enclosed in them. Otherwise, return the same
        * string instance that was passed to this method as the {@code name} parameter.
        *
        * @param   name
        *          Name from which to strip back ticks.
        * @param   dmoName
        *          DMO interface name (for error reporting only).
        *
        * @return  Name without enclosing back ticks.
        *
        * @throws  IllegalArgumentException
        *          if a back tick leads or trails the name, but not both (i.e., unbalanced back tick quotes).
        */
       private static String stripBackTicks(String name, String dmoName)
       {
          String root = name;

          int start = 0;
          int len = name.length();
          boolean backTicks = false;
          boolean unbalanced = false;

          if (name.charAt(0) == '`')
          {
             start++;
             backTicks = true;
          }

          if (name.charAt(len - 1) == '`')
          {
             if (!backTicks)
             {
                unbalanced = true;
             }
             else
             {
                len--;
             }
          }
          else if (backTicks)
          {
             unbalanced = true;
          }

          if (unbalanced)
          {
             throw new IllegalArgumentException("Unbalanced back-ticks in table name for DMO " + dmoName);
          }

          if (backTicks)
          {
             root = name.substring(start, len);
          }

          return root;
       }

       /**
        * @return Clone of the {@code loadSql}, which provides the SQL query statement(s) to access data needed to
        * load a single DMO instance
        */
       public String[] getLoadSql()
       {
          return loadSql.clone();
       }

       /**
        * Enclose a name in back ticks quotes.
        *
        * @param   name
        *          Name to enclose.
        *
        * @return  Name in back tick quotes.
        */
       private static String backTickify(String name)
       {
          return '`' + name + '`';
       }

       /**
        * Obtain the {@code DmoMeta} structure which contains the legacy meta information.
        *
        * @return  {@code DmoMeta} structure which contains the legacy meta information
        */
       public DmoMeta getDmoMeta()
       {
          return dmoMeta;
       }

       /**
        * Indicate whether this object represents metadata for a temporary table.
        *
        * @return  {@code true} if backing, primary, table is a temporary table, else {@code false}.
        */
       public boolean isTemporary()
       {
          return temporary;
       }

       /**
        * Get the array of {@link PropertyMeta property metadata} objects which describe how the core
        * data of this record is organized.
        *
        * @param   nativeOrder
        *          If {@code true} the returned array of properties will be sorted by their
        *          {@code id}-s. This is the 'native' order, in which the records are found in
        *          exported (.d dump) files. This view of the properties is only needed when the
        *          import process is run.<br>
        *          Otherwise, the array is ordered scalar properties first, then the extent properties
        *          grouped by their extent size. Each group represents a series, all series are sorted
        *          by their common extent size. This is the 'persistence' order.
        *
        * @return  Array of property metadata objects.
        */
       public PropertyMeta[] getPropertyMeta(boolean nativeOrder)
       {
          if (nativeOrder)
          {
             PropertyMeta[] nativeOrderProps = Arrays.copyOf(props, props.length);
             Arrays.sort(nativeOrderProps, Comparator.comparingInt(PropertyMeta::getId));
             return nativeOrderProps;
          }
          else
          {
             return props;
          }
       }

       /**
        * Get a bit set representing all DMO properties which participate in indices in the backing
        * table. The 0-based positions of the set bits indicate the 0-based positions of the
        * corresponding data elements in the record's data array.
        *
        * @return  A bitset as described above. The original bit set (not a copy) is returned for maximum
        *          performance. It must not be altered.
        */
       public BitSet getAllIndexedProperties()
       {
          return allIndexedProps;
       }

       /**
        * Get a bit set representing all DMO properties which must be validated when a record is created or
        * updated. These include properties which either are mandatory or which participate in one or more
        * unique indices in the backing table. The 0-based positions of the set bits indicate the 0-based
        * positions of the corresponding data elements in the record's data array.
        *
        * @return  A bitset as described above. The original bit set (not a copy) is returned for maximum
        *          performance. It must not be altered.
        */
       public BitSet getValidatableProperties()
       {
          return validatable;
       }

       /**
        * Get an array of bit sets, each of which describes the components of an unique index.
        * The 0-based positions of the set bits indicate the 0-based positions of the corresponding
        * data elements in the record's data array. Note that this representation only describes
        * which properties participate in an index, but not the precedence of those properties.
        *
        * @return  An array of bit sets as described above. The original bit sets (not copies) are returned
        *          for maximum performance. They must not be altered.
        */
       public BitSet[] getUniqueIndices()
       {
          return uniqueIndices;
       }

       /**
        * Get an array of SQL statements, each of which the tests the uniqueness of a record against an unique
        * index.
        *
        * @return  The set of statements used to check whether a record is unique in the table, according to its
        *          unique indexes (one SELECT statement per such index).
        */
       public String[] getUniqueIndicesValidationSQLs(Dialect d)
       {
          if (uniqueSql == null || d != uniqueSqlDialect)
          {
             if (uniqueSql != null && log.isLoggable(Level.WARNING))
             {
                log.warning("Unique indices validation statements for '" + tables[0] +
                            "' were previously generated for " + uniqueSqlDialect + " dialect (!= " + d + ")");
             }
             // this method depends on [uniqueIndices] to be already set up
             uniqueSql = composeValidationSQLs(tables[0], d);
             uniqueSqlDialect = d;
          }
          return uniqueSql;
       }

       /**
        * Get an array of DELETE SQL statements used to remove existing records which conflict (from the point of
        * view of unique indexes) with a record that is about to be inserted.
        * <p>
        * Only used for FILL operations in REPLACE mode.
        *
        * @return  array of DELETE SQL statements (one for each unique index) that
        */
       public String[] getReplaceCleanupSQLs(Dialect d)
       {
          if (replaceCleanupSql == null || d != replaceCleanupSqlDialect)
          {
             if (replaceCleanupSql != null && log.isLoggable(Level.WARNING))
             {
                log.warning("REPLACE cleanup statements for '" + tables[0] + "' were previously generated for " +
                            replaceCleanupSqlDialect + " dialect (!= " + d + ")");
             }

             // this method depends on [uniqueIndices] to be already set up
             replaceCleanupSql = composeReplaceCleanupSQLs(tables[0], d);
             replaceCleanupSqlDialect = d;
          }
          return replaceCleanupSql;
       }

       /**
        * Get an array of SQL query statements used to extract existing records which conflict (from the point of
        * view of unique indexes) with a record that is about to be inserted.
        * <p>
        * Only used for FILL operations in REPLACE mode.
        *
        * @return  array of DELETE SQL statements (one for each unique index) that
        */
       public String[] getIndexKeyCollisionsSQLs(Dialect d)
       {
          if (getIndexKeyCollisionsSQLs == null || d != getIndexKeyCollisionsDialect)
          {
             if (getIndexKeyCollisionsSQLs != null && log.isLoggable(Level.WARNING))
             {
                log.warning("SELECT Index-Key collision statements for '" + tables[0] +
                            "' were previously generated for " + getIndexKeyCollisionsDialect +
                            " dialect (!= " + d + ")");
             }

             // this method depends on [uniqueIndices] to be already set up
             getIndexKeyCollisionsSQLs = composeFindIndexKeyCollisionsSQL(tables[0], d);
             getIndexKeyCollisionsDialect = d;
          }
          return getIndexKeyCollisionsSQLs;
       }

       /**
        * Get an array of bit sets, each of which describes the components of a non-unique index.
        * The 0-based positions of the set bits indicate the 0-based positions of the corresponding
        * data elements in the record's data array. Note that this representation only describes
        * which properties participate in an index, but not the precedence of those properties.
        *
        * @return  An array of bit sets as described above. The original bit sets (not copies) are returned
        *          for maximum performance. They must not be altered.
        */
       public BitSet[] getNonuniqueIndices()
       {
          return nonuniqueIndices;
       }

       /**
        * Obtain the legacy name of an index, by its order in the list of unique or non-unique lists.
        *
        * @param   pos
        *          The position in the list of unique or non-unique list of indices.
        * @param   unique
        *          If {@code true}, the name of the {@code pos} unique index is returned, otherwise
        *          the non-unique list is used.
        *
        * @return  The legacy name of the specified index, or {@code null} if the index is invalid.
        */
       public String getIndexLegacyName(int pos, boolean unique)
       {
          if (pos < 0)
          {
             return null;
          }

          if (unique)
          {
             if (pos >= uniqueIndicesLegacyNames.length)
             {
                return null;
             }
             return uniqueIndicesLegacyNames[pos];
          }
          else
          {
             if (pos >= nonuniqueIndicesLegacyNames.length)
             {
                return null;
             }
             return nonuniqueIndicesLegacyNames[pos];
          }
       }

       /**
        * Obtain the array of names of unique indices, if any. The order will match the one from
        * {@code getUniqueIndices()} and {@code getIndexLegacyName()}.
        *
        * @return  the array of index names.
        */
       public String[] getUniqueIndexNames()
       {
          return uniqueIndicesNames;
       }

       /**
        * Obtain the array of names of unique indices, if any. The order will match the one from
        * {@code getNonuniqueIndices()} and {@code getIndexLegacyName()}.
        *
        * @return  the array of index names.
        */
       public String[] getNonuniqueIndexNames()
       {
          return nonuniqueIndicesNames;
       }

       /**
        * Obtain the legacy name of the parent table.
        *
        * @return  the legacy name of the parent table.
        */
       public String getLegacyName()
       {
          return legacyName;
       }

       /**
        * Obtain the index of a property in the {@code data} array based on its name.
        * <p>
        * <strong>NOTE:</strong><br>
        * Use this method only in non-time-critical code. It uses map access and usage is not recommended.
        *
        * @param   propName
        *          The name of the property.
        *
        * @return  The base-0 index of the property. If the property is not defined in this DMO, -1 is returned.
        */
       public int getIndexOfProperty(String propName)
       {
          Integer integer = reversedIndexProperties.get(propName);
          return integer == null ? -1 : integer;
       }

       /**
        * Obtain the id of an index using its name.
        *
        * @param   name
        *          The name of the index.
        *
        * @return  The id of the index. If positive, it is an unique index. If negative, it is a non-unique index.
        *          If 0, the index could not be found (invalid index name).
        */
       public int getIndexIdByName(String name)
       {
          Integer indexId = indexIdByName.get(name);
          return indexId == null ? 0 : indexId;
       }

       /**
        * Obtain the index of the primary index.
        *
        * @return  the index of the primary index.
        */
       public int getPrimaryIndexId()
       {
          return primaryIndex;
       }

       /**
        * A helper object which applies the result of an invocation of a dynamic initial value (e.g., "today"
        * for a date, "now" for a datetime[tz]) to one or more data elements in a newly created record. To ensure
        * consistency, the same value is applied to all data elements which specify the same dynamic function.
        */
       private static class DynamicInitializer
       {
          /** Supplier function which provides the initial value. */
          private final Supplier<Object> supplier;

          /** A bit set of the indices of the properties to which the initial value should be applied. */
          private final BitSet properties;

          /**
           * Constructor.
           *
           * @param   supplier
           *          Supplier function which provides the initial value.
           * @param   propSize
           *          The length of the record's data array.
           */
          DynamicInitializer(Supplier<Object> supplier, int propSize)
          {
             this.supplier = supplier;
             this.properties = new BitSet(propSize);
          }

          /**
           * Register a property to receive the supplier's result as its initial data value.
           *
           * @param   index
           *          0-based index of the property in the record's data array.
           */
          void registerProperty(int index)
          {
             properties.set(index);
          }

          /**
           * Invoke the supplier function and store its result in the record's data array for all registered
           * properties.
           *
           * @param   data
           *          Record's data array.
           */
          void apply(Object[] data)
          {
             Object value = supplier.get();
             for (int i = properties.nextSetBit(0); i >= 0; i = properties.nextSetBit(i + 1))
             {
                data[i] = value;
             }
          }
       }
    }