# RecordNursery.java

    /*
    ** Module   : RecordNursery.java
    ** Abstract : A holding site for newly created records which are not yet flushed to the database.
    **
    ** Copyright (c) 2020-2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --------------------------------------Description----------------------------------------
    ** 001 ECF 20200820 Created initial version.
    ** 002 ECF 20200924 Create session if needed.
    **     CA  20201003 Use an identity map where possible.
    **     OM  20220614 Be sure the WRITE trigger is invoked before flushing a record to SQL table, but avoid
    **                  indirect recursive call at the same time.
    **     OM  20221012 The TriggerTracker are bound to DMO, not to the buffer they are contained into.
    **     CA  20221021 Always allow the records to be made visible, just block a possible trigger recursion.
    **     OM  20230111 The dirty context must be notified when a record was flushed to persistence.
    ** 003 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    ** 004 RAA 20230525 Disabled TriggerTracker for TemporaryBuffer.
    ** 005 TJD 20221125 Update DirtyShareContext on every index update
    **     TJD 20240110 Fixes for DirtyShare cross session.
    **     AL2 20240627 Fixed indexUpdated not identifying the primary index as 0.
    **     TJD 20240710 Leak intro-session changes to database, write trigger started only for the first leak
    **     TJD 20240902 Allow checking if RecordNursery holds any DMO related to given RecordBuffer
    **     TJD 20240912 DirtyShareSupport flags logic dependency update
    ** 006 SP  20241029 Updated containsRecordBufferDmo to also check if the indexedRecord map is empty.
    ** 007 EAB 20241008 Created a routine to check for other buffers referencing the same DMO.
    **     AL2 20241023 Fixed conditional to check if DMO is still in use by another buffer.
    **     AL2 20241105 Check isReferenced instead of isInUse, as we are not interested in TRACKED records.
    ** 008 AL2 20240829 Major refactor of the RecordNursery to also manage updated records on a non-index basis.
    **     AL2 20240830 Used update instead of insert when making the records visible through the updates stash.
    **     AL2 20240903 Continued refactoring. Properly integrated dirty data construct to better reflect updates.
    **     AL2 20241025 Added extra checks for cross/intra session dirty share.
    **     AL2 20241106 Properly invalidate ffCache no matter the configuration for the index updated.
    **     AL2 20241112 Get buffer from remove caller instead of inferring from the stored DMOs.
    ** 009 AL2 20241127 Notify the removal attempt of a record from the stash only if it was part of it.
    ** 010 AL2 20241203 Always notify the removal attempt when cross-session dirty-share is active.
    ** 011 AL2 20240110 Don't force dirty-share insert on buffer flush if the record is not already tracked.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist;

    import java.util.*;
    import com.goldencode.p2j.persist.dirty.*;
    import com.goldencode.p2j.persist.lock.LockType;
    import com.goldencode.p2j.persist.orm.*;
    import com.goldencode.p2j.persist.trigger.*;
    import com.goldencode.p2j.util.BaseDataType;
    import com.goldencode.p2j.util.logging.*;

    /**
     * A holding site for new or updated DMOs which are not yet flushed to the primary database. When changes to
     * a new record's data fully update an index, the record is stored here until it is persisted to the database
     * (or such persistence fails). For updated records, any update will register that records in the nursery to
     * delay its inserting in the dirty database as much as possible.
     *
     * When a query is about to be executed on the same index for new records or the same table for updated
     * records, any pending DMOs stored are made visible, either by persisting them to their primary
     * database, or inserting them into their dirty database.
     */
    public final class RecordNursery
    {
       /** Logger */
       private static final CentralLogger LOG = CentralLogger.get(RecordNursery.class);

       /** Stash of records that were registered to the nursery as new */
       private RecordsStash<IndexedKey> newRecords;

       /** Stash of records that were registered to the nursery as updated, but are not transient. */
       private RecordsStash<Key> updatedRecords;

       /** Fast-find cache, which must be invalidated when an index is updated, to prevent stale results */
       private final FastFindCache ffCache;

       /** {@code true} if the cross-session dirty share functionality is enabled */
       private final boolean crossSessionEnabled;

       /** {@code true} if the intra-session dirty share functionality is enabled */
       private final boolean intraSessionEnabled;

       /**
        * Convenient method to retrieve a {@link DirtyData} object for an existing set of dirty props.
        * If the dirty props is not provided, the one from the dmo will be inferred. This is decided
        * due to the cases where dmo was freshly flushed and there are no dirty props anymore. In that case
        * the dirty props are passed manually on the last parameter. Based on this dirty data, the dirty
        * database will do the update.
        *
        * @param    buffer
        *           The buffer that was holding the dmo when the flushing was triggered.
        * @param    dmo
        *           The dmo whose dirty data should be gathered.
        * @param    dirtyProps
        *           The properties that were known to be dirty before flush, or {@code null} if this was not
        *           triggered by a flush and the dirty props can be retrieved from the dmo.
        *
        * @return   A dirty data instance used for inserting/updating DMOs in the dirty database.
        */
       static DirtyData getDirtyData(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
       {
          if (dirtyProps == null)
          {
             dirtyProps = dmo.getDirtyProps();
          }

          if (dirtyProps == null || dirtyProps.size() == 0)
          {
             return null;
          }

          DmoMeta dmoMeta = buffer.getDmoInfo();
          RecordMeta metadata = dmoMeta.getRecordMeta();
          PropertyMeta[] propertyMeta = metadata.getPropertyMeta(false);

          // recalculate the size, taking into considerations the arrays
          int newSize = dirtyProps.cardinality();

          String[]       dup = new String[newSize];       // property name
          int[]          dui = new int[newSize];          // extent offset (-1 for scalars)
          BaseDataType[] duv = new BaseDataType[newSize]; // the values

          for (int i = 0, j = dirtyProps.nextSetBit(0); j >= 0; j = dirtyProps.nextSetBit(j + 1), i++)
          {
             PropertyMeta pm = propertyMeta[j];
             dup[i] = pm.getName();
             dui[i] = pm.getExtent() == 0 ? -1 : j - pm.getOffset();
             duv[i] = pm.getDataHandler().getField((Record) dmo, j);
          }

          return new DirtyData(dup, dui, duv);
       }

       /**
        * Default constructor.
        *
        * @param   ffCache
        *          The fast-find cache that should be sometimes invalidated by this nursery.
        */
       public RecordNursery(FastFindCache ffCache)
       {
          this.ffCache = ffCache;
          this.newRecords = new TransientRecordsStash(this, ffCache);
          this.updatedRecords = new UpdatedRecordsStash(this, ffCache);
          this.crossSessionEnabled = DirtyShareSupport.isEnabledCrossSession();
          this.intraSessionEnabled = DirtyShareSupport.isEnabledIntraSession();
       }

       /**
        * A record was just updated, but its state is not flushed yet to avoid any unwanted write triggers.
        * Store the corresponding DMO in the nursery until it is ready to be flushed into the database.
        *
        * @param   buffer
        *          Record buffer containing the record.
        * @param   dmo
        *          Record to be stored.
        * @throws  PersistenceException
        *          if there is any error persisting the record to its primary or dirty database.
        */
       public void indexUpdated(RecordBuffer buffer, BaseRecord dmo)
       throws PersistenceException
       {
          int dmoUid = buffer.getDmoInfo().getId();
          Integer mpxID = buffer.getMultiplexID();

          if (this.shouldForceDirty(buffer))
          {
             DirtyShareContext dctx = buffer.getDirtyContext();
             // track the record into the dirty share context
             dctx.insert(buffer, (Record) dmo, true, null);
          }
          else if (this.shouldRetainInNursery(buffer))
          {

             Key key = new Key(dmoUid, mpxID);
             updatedRecords.recordChanges(buffer, dmo, key);
          }

          ffCache.invalidate(dmoUid, mpxID);
       }

       /**
        * A set of indexes were updated by changes to a newly created, as yet unvalidated record. Store
        * the corresponding DMO in the nursery until it is ready to be inserted into the database.
        * Make change visible via DirtyShareContext if needed - call DirtyShare only once for all indexes
        *
        * @param   buffer
        *          Record buffer containing the record.
        * @param   indices
        *          Bit set representing those indices that were updated. If <b>null</b> update only primary index.
        * @param   unique
        *          {@code true} if {@code indices} refers to unique indices; else {@code false}.
        * @param   dmo
        *          Record to be stored.
        * @throws  PersistenceException
        *          if there is any error persisting the record to its primary or dirty database.
        */
       public void indexUpdated(RecordBuffer buffer, BitSet indices, boolean unique, BaseRecord dmo)
       throws PersistenceException
       {
          int dmoUid = buffer.getDmoInfo().getId();
          Integer mpxID = buffer.getMultiplexID();

          if (this.shouldForceDirty(buffer))
          {
             DirtyShareContext dctx = buffer.getDirtyContext();
             // track the record into the dirty share context
             dctx.insert(buffer, (Record) dmo, true, null);
          }
          else if (this.shouldRetainInNursery(buffer))
          {
             if (indices == null)
             {
                IndexedKey key = new IndexedKey(dmoUid, mpxID, 0);
                newRecords.recordChanges(buffer, dmo, key);
             }
             else
             {
                for (int i = indices.nextSetBit(0); i >= 0; i = indices.nextSetBit(i + 1))
                {
                   int idxUid = (i + 1) * (unique ? 1 : -1);
                   // indexUpdated can be called for non-transient records, so check the buffer state first
                   IndexedKey key = new IndexedKey(dmoUid, mpxID, idxUid);
                   newRecords.recordChanges(buffer, dmo, key);
                }
             }
          }

          if (indices == null)
          {
             ffCache.invalidate(dmoUid, mpxID, 0);
          }
          else
          {
             for (int i = indices.nextSetBit(0); i >= 0; i = indices.nextSetBit(i + 1))
             {
                int idxUid = (i + 1) * (unique ? 1 : -1);
                ffCache.invalidate(dmoUid, mpxID, idxUid);
             }
          }
       }

       /**
        * The current transaction ended; clean up data structures.
        */
       public void transactionEnded()
       {
          newRecords.clear();
          updatedRecords.clear();
       }

       /**
        * The given record was flushed to the database or determined to be invalid; remove it from the nursery.
        *
        * @param   buffer
        *          The buffer that determined the remove
        * @param   dmo
        *          The record to be removed.
        * @throws  PersistenceException
        *          the removal may flush some information to the dirty manager, so it may cause persistence
        *          issues.
        */
       public void remove(RecordBuffer buffer, BaseRecord dmo)
       throws PersistenceException
       {
          remove(buffer, dmo, null);
       }

       /**
       * The given record was flushed to the database or determined to be invalid; remove it from the nursery.
       *
       * @param   buffer
       *          The buffer that determined the remove
       * @param   dmo
       *          The record to be removed.
       * @param   dirtyProps
       *          Dirty properties before the dmo was actually validated and flushed. This is explicitly sent
       *          because we can't infer the dirty props from the DMO anymore.
       *
       * @throws  PersistenceException
       *          the removal may flush some information to the dirty manager, so it may cause persistence
       *          issues.
       */
       public void remove(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
       throws PersistenceException
       {
          newRecords.remove(buffer, dmo, dirtyProps);
          updatedRecords.remove(buffer, dmo, dirtyProps);
       }

       /**
        * Check if RecordNursery holds any DMO related to given RecordBuffer
        *
        * @param   buffer
        *          Record buffer containing the record.
        *
        * @return  {@code true} if there is at least one DMO related to given RecordBuffer
        */
       public boolean containsRecordBufferDmo(RecordBuffer buffer)
       {
          int dmoUid = buffer.getDmoInfo().getId();
          Integer multiplex = buffer.getMultiplexID();
          int nonUniqueIndexesCount = buffer.getDmoInfo().getIndexCount(false);
          int uniqueIndexesCount = buffer.getDmoInfo().getIndexCount(true);

          Key key = new Key(dmoUid, multiplex);
          if (updatedRecords.containsKeyNotEmpty(key))
          {
             return true;
          }

          for (int idxUid = -nonUniqueIndexesCount; idxUid <= uniqueIndexesCount; idxUid++)
          {
             IndexedKey indexedKey = new IndexedKey(dmoUid, multiplex, idxUid);
             if (newRecords.containsKeyNotEmpty(indexedKey))
             {
                return true;
             }
          }

          return false;
       }

       /**
        * Make all records which have been stored in the nursery on a particular index visible
        * (searchable). If the record is a temp-table record (whether valid or not) or a persistent record
        * which is determined to be valid, persist it to its database. For persistent records which have not
        * yet been fully validated, insert it into the dirty share manager.
        *
        * @param   dmoUid
        *          Unique DMO type/table identifier.
        * @param   multiplex
        *          Optional multiplex ID (temp-tables only); otherwise {@code null}.
        * @param   idxUid
        *          Index identifier (unique within DMO type/table).
        *
        * @throws  PersistenceException
        *          if there is any error persisting the record to its primary or dirty database.
        */
       public void makeVisible(int dmoUid, Integer multiplex, int idxUid)
       throws PersistenceException
       {
          boolean invalidateCache1 = newRecords.makeVisible(new IndexedKey(dmoUid, multiplex, idxUid));
          boolean invalidateCache2 = updatedRecords.makeVisible(new Key(dmoUid, multiplex));

          if (invalidateCache1 || invalidateCache2)
          {
             ffCache.invalidate(dmoUid, multiplex);
          }
       }

       /**
        * Check if the record nursery registration should be skipped for this buffer according to the
        * cross/intra session dirty configuration. Temporary records should never go into the cross-dirty
        * share manager, while the persistent records need to be forced to go there only if the
        * cross-dirty is enabled.
        * <p>
        * Such strategy is needed to avoid any delay of reporting changes to the cross-dirty share manager.
        * An intra-session scenario has the luxury to cache the changes here and report them only when
        * needed.
        *
        * @param   buffer
        *          The buffer for which the check should be done.
        *
        * @return  {@code true} if the buffer changes should bypass the record nursery caching and be
        *          made visible into the dirty share manager right away.
        */
       private boolean shouldForceDirty(RecordBuffer buffer)
       {
          if (buffer.isTemporary())
          {
             return false;
          }

          return this.crossSessionEnabled && buffer.getDirtyContext() != null;
       }

       /**
        * Check if the record nursery registration is required for this buffer according to the
        * cross/intra session dirty configuration. Temporary records should go in the record
        * nursery only if the intra-session configuration is enabled. For persistent records,
        * we shall register only if intra-session dirty share is enabled, but the cross one
        * is disabled. If the cross-session dirty share is enabled, then the changes are
        * already eagerly reflected in the dirty database - no need for caching.
        *
        * @param   buffer
        *          The buffer for which the check should be done.
        *
        * @return  {@code true} if the buffer changes should be retained the record nursery caching
        *          and be made visible into the dirty share manager at the right time..
        */
       private boolean shouldRetainInNursery(RecordBuffer buffer)
       {
          if (buffer.isTemporary())
          {
             return this.intraSessionEnabled;
          }
          else
          {
             return this.intraSessionEnabled && !this.crossSessionEnabled;
          }
       }

       /**
        * A collection of records that were not physically flushed to the primary database yet to avoid
        * unwanted unique index violations or write trigger firing. These should theoretically reside in the
        * dirty database, but are stored here to delay the dirty database work as much as possible for the
        * sake of performance.
        *
        * @param   <K>
        *          The type of key that will group the records
        */
       private static abstract class RecordsStash<K extends Key>
       {
          /** The nursery associated with this stash. */
          final RecordNursery nursery;

          /** Maps of records to buffers, organized by a specific key. */
          final Map<K, Map<BaseRecord, RecordBuffer>> records = new HashMap<>();

          /** Map of record to keys in {@link #records} map, to allow faster cleanup of the latter */
          final Map<BaseRecord, Set<K>> keysByDmo = new IdentityHashMap<>();

          /** The fast find cache that may be subject to changes with work from this stash. */
          final FastFindCache ffCache;

          /**
           * Basic constructor.
           *
           * @param   nursery
           *          The record nursery associated with this stash.
           * @param   ffCache
           *          The ff cache that may be subject to invalidation after working with the stash.
           */
          public RecordsStash(RecordNursery nursery, FastFindCache ffCache)
          {
             this.nursery = nursery;
             this.ffCache = ffCache;
          }

          /**
           * Check if this stash contains a certain key.
           * @param   key
           *          The key to be checked.
           * @return  {@code true} if the provided key is managed by this stash.
           */
          public boolean containsKey(K key)
          {
             return records.containsKey(key);
          }

          /**
           * Check if this stash contains a certain key and is not empty.
           * @param   key
           *          The key to be checked.
           * @return  {@code true} if the provided key is managed by this stash.
           */
          public boolean containsKeyNotEmpty(K key)
          {
             Map<BaseRecord, RecordBuffer> record = records.get(key);
             return record != null && !record.isEmpty();
          }

          /**
           * A record was just updated, but its state is not flushed yet to avoid any unwanted behavior.
           * Register the change to the stash until it is ready to be flushed into the database.
           *
           * @param   buffer
           *          Record buffer containing the record.
           * @param   dmo
           *          Record to be stored.
           * @param   key
           *          The key that defines the group in which the record belongs.
           */
          public void recordChanges(RecordBuffer buffer, BaseRecord dmo, K key)
          {
             Map<BaseRecord, RecordBuffer> dmos = records.computeIfAbsent(key, k -> new IdentityHashMap<>());
             dmos.put(dmo, buffer);

             Set<K> keys = keysByDmo.computeIfAbsent(dmo, k -> new HashSet<>());
             keys.add(key);
          }

          /**
           * Make all records stored in this stash visible (searchable).
           * <ul>
           *    <li>If the record is a temp-table record (whether valid or not) or a persistent record
           *        which is determined to be valid, persist it to its database.</li>
           *    <li>For persistent records which have not yet been fully validated,
           *        insert it into the dirty share manager.</li>
           * </ul>
           *
           * @param   key
           *          The key that defines the group in which the record belongs.
           *
           * @return  {@code true} if the ff cache should be invalidated.
           *
           * @throws  PersistenceException
           *          if there is any error persisting the record to its primary or dirty database.
           */
          public abstract boolean makeVisible(K key)
          throws PersistenceException;

          /**
           * This is called when a dmo is removed from the record nursery. This can happen either because the
           * record was flush into the primary database or it was actually deleted. If it was deleted, for sure
           * it should be removed from the nursery to avoid phantom records. If it was flushed, then don't insist
           * on making it available through the dirty share manager if not already tracked.
           *
           * @param   buffer
           *          The buffer that caused the create/update in the first place, registering the dmo in
           *          this nursery.
           * @param   dmo
           *          The record that was removed from the nursery.
           * @param   dirtyProps
           *          Dirty properties before the dmo was actually validated and flushed. This is explicitly sent
           *          because we can't infer the dirty props from the DMO anymore.
           *
           * @throws  PersistenceException
           *          the removal may flush some information to the dirty manager, so it may cause persistence
           *          issues.
           */
          public abstract void notifyRemoved(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
          throws PersistenceException;

          /**
           * The given record was flushed to the database or determined to be invalid; remove it from the nursery.
           *
           * @param   buffer
           *          The buffer that determined the remove
           * @param   dmo
           *          The record to be removed.
           * @param   dirtyProps
           *          Dirty properties before the dmo was actually validated and flushed. This is explicitly sent
           *          because we can't infer the dirty props from the DMO anymore.
           *
           * @throws  PersistenceException
           *          the removal may flush some information to the dirty manager, so it may cause persistence
           *          issues.
           */
          public void remove(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
          throws PersistenceException
          {
             boolean forceNotify = false;
             if (nursery.shouldRetainInNursery(buffer))
             {
                Set<K> keys = keysByDmo.remove(dmo);
                if (keys != null && !keys.isEmpty())
                {
                   for (K key : keys)
                   {
                      Map<BaseRecord, RecordBuffer> map = records.get(key);
                      if (map != null)
                      {
                         map.remove(dmo);
                      }
                   }

                   forceNotify = true;
                }
             }

             if (dirtyProps != null && (forceNotify || nursery.shouldForceDirty(buffer)))
             {
                notifyRemoved(buffer, dmo, dirtyProps);
             }
          }

          /**
           * The current transaction ended; clean up data structures.
           */
          public void clear()
          {
             records.clear();
             keysByDmo.clear();
          }

          /**
           * Useful method that saved a dmo to the primary database. This should be carefully used by makeVisible
           * methods to flush a record directly to the primary database if possible.
           *
           * @param   buffer
           *          The buffer that holds the dmo which should be flushed.
           * @param   dmo
           *          The dmo that should be flushed
           * @param   wasNew
           *          {@code true} if this was called to flush a newly created record for the first time.
           *
           * @return  {@code true} if the fast find cache should be invalidated.
           *
           * @throws  PersistenceException
           *          The method is flushing to the database; this can pop up.
           */
          boolean saveDmo(RecordBuffer buffer, BaseRecord dmo, boolean wasNew)
          throws PersistenceException
          {
             boolean invalidateCache = false;

             // then save the (possibly invalid) record to the database
             Session session = buffer.getSession(true);
             boolean needTx = !session.inTransaction();
             try
             {
                if (needTx)
                {
                   session.beginTransaction();
                }

                session.save(dmo, true);

                if (needTx)
                {
                   session.commit();
                }

                invalidateCache = true;
                if (wasNew)
                {
                   buffer.recordInserted();
                   buffer.markPersisted();
                }

                buffer.reportChange((Record) dmo, wasNew, false);
             }
             catch (PersistenceException exc)
             {
                LOG.warning("", exc);
                if (needTx)
                {
                   session.rollback();
                }
             }

             return invalidateCache;
          }

          /**
           * Gateway to the dirty share manager. This should be used when the record should be propagated to the
           * underlying dirty database: either due to a need to make visible or because it was requested to be
           * removed from the nursery. TODO: move to updated records stash
           *
           * @param   buffer
           *          The buffer that caused the updated dmo to be updated.
           * @param   dmo
           *          The dmo that was updated.
           * @param   dirtyProps
           *          Dirty properties before the dmo was actually validated and flushed. This is explicitly sent
           *          because we can't infer the dirty props from the DMO anymore.
           *
           *
           * @throws  PersistenceException
           *          the removal may flush some information to the dirty manager, so it may cause persistence
           *          issues.
           */
          void updateDirty(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
          throws PersistenceException
          {
             String entity = buffer.getEntityName();
             DirtyShareContext dirtyContext = buffer.getDirtyContext();

             if (dirtyContext == null)
             {
                return;
             }

             try
             {
                // lock all indexes while sharing (and validating, if needed)
                // TODO: optimization: lock only dirty indexes unless we're inserting a new record. As part of this
                //       optimization, we need to consider the remote server case, such that we minimize the data
                //       sent across (an array or collections of index name strings may not be efficient).
                dirtyContext.lockAllIndexes(entity, LockType.EXCLUSIVE);

                DirtyData dirtyData = getDirtyData(buffer, dmo, dirtyProps);

                if (dirtyData != null)
                {
                   dirtyContext.update(buffer.getEntityName(), (Record) dmo, false, dirtyData);

                   DmoMeta dmoMeta = buffer.getDmoInfo();
                   ffCache.invalidate(dmoMeta.getId(), buffer.getMultiplexID());
                }
             }
             finally
             {
                 // Unlock unique indexes.  See TODO above.
                 dirtyContext.lockAllIndexes(entity, LockType.NONE);
             }
          }
       }

       /**
        * Stash dedicated for transient records that were registered with the nursery. These are special in the
        * sense that they are made visible "per-index", not entirely. So the key is honoring the index on which
        * the record should be visible.
        */
       private static class TransientRecordsStash
       extends RecordsStash<IndexedKey>
       {
          /**
           * Basic constructor.
           *
           * @param   nursery
           *          The record nursery associated with this stash.
           * @param   ffCache
           *          The ff cache that may be subject to invalidation after working with the stash.
           */
          TransientRecordsStash(RecordNursery nursery, FastFindCache ffCache)
          {
             super(nursery, ffCache);
          }

          /**
           * Make all records stored in this stash visible (searchable).
           * <ul>
           *    <li>If the record is a temp-table record (whether valid or not) or a persistent record
           *        which is determined to be valid, persist it to its database.</li>
           *    <li>For persistent records which have not yet been fully validated,
           *        insert it into the dirty share manager.</li>
           * </ul>
           *
           * @param   key
           *          The key that defines the group in which the record belongs.
           *
           * @return  {@code true} if the ff cache should be invalidated.
           *
           * @throws  PersistenceException
           *          if there is any error persisting the record to its primary or dirty database.
           */
          @Override
          public boolean makeVisible(IndexedKey key)
          throws PersistenceException
          {
             Map<BaseRecord, RecordBuffer> dmos = records.get(key);

             if (dmos == null || dmos.isEmpty())
             {
                return false;
             }

             boolean invalidateCache = false;

             try
             {
                for (Map.Entry<BaseRecord, RecordBuffer> entry : dmos.entrySet())
                {
                   BaseRecord dmo = entry.getKey();
                   RecordBuffer buffer = entry.getValue();
                   if (buffer.getCurrentRecord() == null)
                   {
                      if (dmo.isReferenced())
                      {
                         buffer = buffer.getBufferManager()
                                        .getRecordBuffersByDMOBufClass(buffer.getDMOBufInterface().getName(),
                                                                       buffer.getMultiplexID(),
                                                                       dmo.primaryKey());

                         if (buffer == null)
                         {
                            continue;
                         }
                      }
                      else
                      {
                         continue;
                      }
                   }

                   Set<IndexedKey> keys = keysByDmo.get(dmo);
                   if (keys != null)
                   {
                      keys.remove(key);
                   }

                   boolean temp = buffer.isTemporary();
                   BitSet dirtyProps = null;
                   if (temp || !dmo.needsValidation())
                   {
                      if ((buffer.getTriggerTracker() == null ||
                          !buffer.getTriggerTracker()
                                 .isExecuting(DatabaseEventType.WRITE, buffer.getCurrentRecord().primaryKey())))
                      {
                         // avoid indirect recursion. This buffer is already about to be flushed up the stack
                         // invoke the Write trigger, if necessary, but only for the first time record is flushed
                         // consider other flushes as local cross-buffers leaks
                         buffer.maybeFireWriteTrigger();
                      }

                      // temp doesn't use dirty share; don't snapshot the dirty props
                      dirtyProps = temp ? null : (BitSet) dmo.getDirtyProps().clone();
                      invalidateCache = saveDmo(buffer, dmo, true);
                   }
                   // in case dmo needs validation store it in the dirty db to allow correct querying via dirty DB
                   else
                   {
                      DirtyShareContext dctx = buffer.getDirtyContext();

                      if (dctx != null)
                      {
                         // insert the record into the dirty share context
                         dctx.insert(buffer, (Record) dmo, true, null);
                         invalidateCache = true;
                      }
                   }

                   remove(buffer, dmo, dirtyProps);
                }
             }
             finally
             {
                dmos.clear();
             }

             return invalidateCache;
          }

          /**
           * This is called when a dmo is removed from the record nursery. This can happen either because the
           * record was flush into the primary database or it was actually deleted. If it was deleted, for sure
           * it should be removed from the nursery to avoid phantom records. If it was flushed, then don't insist
           * on making it available through the dirty share manager if not already tracked.
           *
           * @param   buffer
           *          The buffer that caused the create/update in the first place, registering the dmo in
           *          this nursery.
           * @param   dmo
           *          The record that was removed from the nursery.
           * @param   dirtyProps
           *          Dirty properties before the dmo was actually validated and flushed. This is explicitly sent
           *          because we can't infer the dirty props from the DMO anymore.
           *
           * @throws  PersistenceException
           *          the removal may flush some information to the dirty manager, so it may cause persistence
           *          issues.
           */
          @Override
          public void notifyRemoved(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
          throws PersistenceException
          {
             DirtyShareContext dctx = buffer.getDirtyContext();
             if (dirtyProps != null && dirtyProps.cardinality() > 0 && dctx != null)
             {
                boolean isTracked = dctx.isTracked(new RecordIdentifier<>(buffer.getEntityName(),
                                                                          dmo.primaryKey()));
                if (!isTracked && !nursery.shouldForceDirty(buffer))
                {
                   // if the record is not tracked, don't start tracking it for nothing
                   return;
                }

                DirtyData dirtyData = getDirtyData(buffer, dmo, dirtyProps);

                // insert the record anyway as it should be visible downstream (with or without dirty changes)
                dctx.insert(buffer, (Record) dmo, true, dirtyData);
             }
          }
       }

       /**
        * Stash dedicated for updated records that were registered with the nursery. These are shashed here because
        * we couldn't afford flushing them to the database yet, mostly due to write triggers. Delay the introduction
        * to the dirty database as much as possible.
        */
       private static class UpdatedRecordsStash
       extends RecordsStash<Key>
       {
          /**
           * Basic constructor.
           *
           * @param   nursery
           *          The record nursery associated with this stash.
           * @param   ffCache
           *          The ff cache that may be subject to invalidation after working with the stash.
           */
          UpdatedRecordsStash(RecordNursery nursery, FastFindCache ffCache)
          {
             super(nursery, ffCache);
          }

          /**
           * Make all records stored in this stash visible (searchable).
           * <ul>
           *    <li>If the record is a temp-table record (whether valid or not) or a persistent record
           *        which is determined to be valid, persist it to its database.</li>
           *    <li>For persistent records which have not yet been fully validated,
           *        insert it into the dirty share manager.</li>
           * </ul>
           *
           * @param   key
           *          The key that defines the group in which the record belongs.
           *
           * @return  {@code true} if the ff cache should be invalidated.
           *
           * @throws  PersistenceException
           *          if there is any error persisting the record to its primary or dirty database.
           */
          @Override
          public boolean makeVisible(Key key)
          throws PersistenceException
          {
             Map<BaseRecord, RecordBuffer> dmos = records.get(key);

             if (dmos == null || dmos.isEmpty())
             {
                return false;
             }

             boolean invalidateCache = false;

             try
             {
                for (Map.Entry<BaseRecord, RecordBuffer> entry : dmos.entrySet())
                {
                   BaseRecord dmo = entry.getKey();
                   RecordBuffer buffer = entry.getValue();
                   if (buffer.getCurrentRecord() == null)
                   {
                      if (dmo.isReferenced())
                      {
                         buffer = buffer.getBufferManager()
                                        .getRecordBuffersByDMOBufClass(buffer.getDMOBufInterface().getName(),
                                                                       buffer.getMultiplexID(),
                                                                       dmo.primaryKey());

                         if (buffer == null)
                         {
                            continue;
                         }
                      }
                      else
                      {
                         continue;
                      }
                   }

                   Set<Key> keys = keysByDmo.get(dmo);
                   if (keys != null)
                   {
                      keys.remove(key);
                   }

                   boolean temp = buffer.isTemporary();
                   if (temp)
                   {
                      invalidateCache = saveDmo(buffer, dmo, false);
                   }
                   else
                   {
                      // in case dmo needs validation store it in the dirty db to allow correct querying via dirty DB
                      updateDirty(buffer, dmo, null);
                   }

                   // the DMO is removed from keysByDmo, so we should forcefully update it in the dirty-share
                   remove(buffer, dmo, null);
                }
             }
             finally
             {
                dmos.clear();
             }

             return invalidateCache;
          }

          /**
           * The given record was flushed to the database or determined to be invalid; remove it from the nursery.
           *
           * @param   dmo
           *          The record to be removed.
           * @param   dirtyProps
           *          Dirty properties before the dmo was actually validated and flushed. This is explicitly sent
           *          because we can't infer the dirty props from the DMO anymore.
           *
           * @throws  PersistenceException
           *          the removal may flush some information to the dirty manager, so it may cause persistence
           *          issues.
           */
          @Override
          public void notifyRemoved(RecordBuffer buffer, BaseRecord dmo, BitSet dirtyProps)
          throws PersistenceException
          {
             DirtyShareContext dctx = buffer.getDirtyContext();

             if (dctx != null &&
                 (dirtyProps != null && dirtyProps.cardinality() != 0) &&
                 dctx.isTracked(new RecordIdentifier<>(buffer.getEntityName(), dmo.primaryKey())))
             {
                updateDirty(buffer, dmo, dirtyProps);
             }
          }
       }

       /**
        * The key that groups the records in the stash, based on the DMO id and eventually the multiplex.
        * This is used for updated records as they are visible on all indexes at any time.
        */
       private static class Key
       {
          /** Unique DMO type identifier */
          final long dmoUid;

          /** Cached hash code */
          int hash;

          /**
           * Basic constructor.
           *
           * @param   duid
           *          The id of the DMO
           * @param   multiplex
           *          The multiplex (if any) of the DMO.
           */
          Key(int duid, Integer multiplex)
          {
             this.dmoUid = multiplex == null ? duid : ((((long) multiplex) << 32) | duid);
          }

          /**
           * Simple method that computes the hash and caches it
           *
           * @return   The hash of this key.
           */
          int computeHash()
          {
             return this.hash = Long.hashCode(dmoUid);
          }

          /**
           * Basic equals implementation
           */
          @Override
          public boolean equals(Object o)
          {
             Key that = (Key) o;

             return this.dmoUid == that.dmoUid;
          }

          /**
           * Basic hashCode implementation
           */
          @Override
          public int hashCode()
          {
             return hash != 0 ? hash : computeHash();
          }

          /**
           * Basic toString implementation
           */
          @Override
          public String toString()
          {
             return "Key [dmoUid=" + dmoUid + "," + ", hash=" + hash + "]";
          }
       }

       /**
        * The key that groups the records in the stash, based on the DMO id, eventually the multiplex and
        * the id of an index. This is used for new records as they are visible only on fully updated indexes.
        * Thus, it is important to ensure that only records on specific indexes are made visible.
        */
       private static class IndexedKey
       extends Key
       {
          /** Unique index identifier */
          private final int idxUid;

          /**
           * Basic constructor.
           *
           * @param   duid
           *          The id of the DMO
           * @param   multiplex
           *          The multiplex (if any) of the DMO.
           * @param   iuid
           *          The id of the index used for this key.
           */
          IndexedKey(int duid, Integer multiplex, int iuid)
          {
             super(duid, multiplex);
             this.idxUid = iuid;
          }

          /**
           * Simple method that computes the hash and caches it
           *
           * @return   The hash of this key.
           */
          int computeHash()
          {
             return this.hash = (int) (31 * super.computeHash() + idxUid);
          }

          /**
           * Basic equals implementation
           */
          @Override
          public boolean equals(Object o)
          {
             IndexedKey that = (IndexedKey) o;

             return super.equals(o) && this.idxUid == that.idxUid;
          }

          /**
           * Basic toString implementation
           */
          @Override
          public String toString()
          {
             return "IndexedKey [dmoUid=" + dmoUid + ", idxUid=" + idxUid + ", hash=" + hash + "]";
          }
       }
    }