# ProgressiveResults.java

    /*
    ** Module   : ProgressiveResults.java
    ** Abstract : Results implementation which brackets query results
    **
    ** Copyright (c) 2004-2024, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
    ** 001 ECF 20060725   @28185 Created initial version. Bracketed result set
    **                           implementation.
    ** 002 ECF 20061006   @30197 Rewrote implementation. We no longer keep a
    **                           list of delegate result sets, since only the
    **                           newest one can be used anyway. The class now
    **                           supports scrolling mode. In this mode, a
    **                           cache of results is maintained, so that
    **                           non-linear scrolling is possible.
    ** 003 ECF 20061016   @30529 Fixed algorithm which defines a Locator for
    **                           an arbitrary result set bracket. This
    **                           algorithm is used in non-linear scrolling
    **                           mode.
    ** 004 ECF 20061101   @30896 Fixed implementation of last(). Cursor was
    **                           being positioned at the second to last
    **                           record.
    ** 005 ECF 20061129   @31478 Fixed implementation of last() (again). Was
    **                           not functioning properly in the event that
    **                           the cursor was already positioned at the last
    **                           record in the result set.
    ** 006 ECF 20070319   @32545 More restrictive logging for non-fatal error.
    **                           closeDelegate() method may have a non-fatal
    **                           error while cleaning up. Log error message as
    **                           a warning and full stack trace only if debug
    **                           logging is enabled.
    ** 007 ECF 20070416   @33027 Integrated user interrupt handling.
    ** 008 ECF 20070514   @33480 Fixed cleanup() method. Reset additional
    **                           variables to their default values.
    ** 009 ECF 20070516   @33671 Modified exception handling. Instead of
    **                           delegating errors to the ErrorManager, we now
    **                           log exceptions and use return codes to handle
    **                           control flow.
    ** 010 ECF 20070711   @34437 Added safety code to get(int). This avoids a
    **                           possible NPE in the Hibernate implementation
    **                           of ScrollableResults. We also eagerly close
    **                           the delegate ScrollableResults at the first
    **                           invalid result. This prevents later problems
    **                           when scope-triggered cleanup tries to close
    **                           an already invalid delegate.
    ** 011 ECF 20070711   @34446 Added sessionClosing() method. Caches the
    **                           remaining records in the current results
    **                           bracket, if any, in response to a session
    **                           close event, so that they may still be
    **                           retrieved without requiring a new query to be
    **                           executed.
    ** 012 ECF 20070927   @35275 Fixed off-end behavior. We were not properly
    **                           remembering the cursor position when running
    **                           off the end of the query's results in the
    **                           forward direction. This should be set to the
    **                           last valid position plus one. It was being
    **                           set to -1, which caused ambiguity with the
    **                           case where we run off-end in the backward
    **                           direction (scrolling queries only).
    ** 013 ECF 20071027   @35587 Fixed defect in moveTo(). In cases where no
    **                           records were found at all, this method was
    **                           indicating failure, but leaving the internal
    **                           state at the wrong position. Subsequent calls
    **                           that depended upon this state would be
    **                           unreliable.
    ** 014 SVL 20071230   @36620 To detect gaps created by another context
    **                           deleting records while the query is active,
    **                           ProgressiveResults retrieves an extra record
    **                           at the front end of each bracket, compares
    **                           this record with the last in the previous
    **                           bracket and notifies listeners in case
    **                           of mismatching.
    ** 015 ECF 20080310   @37432 Fixed subtle memory leak introduced with #014
    **                           (@36620). CA discovered that in getResults(),
    **                           the lastRecord check leaves DMOs in the
    **                           Hibernate session. These must be evicted when
    **                           no longer in use.
    ** 016 CA  20080318   @37530 Added an upper limit to the bracket size:
    **                           MAX_EXPONENT will limit the bracket size to
    **                           10K records fetched at a time. This limit is
    **                           needed because the PG JDBC driver will cache
    **                           all the records in binary form, in the computed
    **                           result set.
    ** 017 ECF 20080709   @39087 Modified sessionClosing(). Made short circuit
    **                           check slightly more specific.
    ** 018 ECF 20080729   @39326 Removed assumption that DMO ID is an Integer.
    **                           Use Serializable instead.
    ** 019 SVL 20080815   @39399 reset() is allowed if we are off the front
    **                           end of the results or have only retrieved the
    **                           first set of results.
    ** 020 CA  20080815   @39464 Support API change in DBUtils.
    ** 021 ECF 20090221   @41445 Modified call to BufferManager. Signature of
    **                           decrementDMOUseCount() has changed.
    ** 022 ECF 20090310   @41503 Optimizations to reduce the number of result
    **                           brackets. Increased bracket base to 16 and
    **                           raised MAX_EXPONENTS. This change presumes a
    **                           reasonable JDBC fetch size is configured.
    ** 023 ECF 20090610   @42636 Optimized getResults(). Call BufferManager's
    **                           evictDMOIfUnused() method instead of incrementing
    **                           then decrementing DMO use counts.
    ** 024 ECF 20090617   @42768 Rolled back #022 (@41503). JDBC fetch size alone
    **                           will not necessarily prevent some JDBC drivers
    **                           from failing to use server-side cursors and limit
    **                           the number of results fetched, without more
    **                           involved code changes.
    ** 025 ECF 20090711   @43191 Use new ScrollingResults c'tor.
    ** 026 ECF 20090724   @43420 Added implicit transaction support. We call
    **                           Persistence.pushImplicitTransaction() during
    **                           construction, and popImplicitTransaction() when
    **                           the results are closed.
    ** 027 ECF 20090919   @44063 Fixed sessionClosing(). Do not attempt to cache
    **                           results if this object has been closed. Handle
    **                           HibernateException when advancing to next result
    **                           during caching loop.
    ** 028 SVL 20140812          Fixed last() function: re-initialize delegate after we had gone
    **                           off-end while trying to find the last row. Fixed the case where
    **                           previous should be treated as last.
    ** 029 ECF 20150721          Added support for multi-table results; increased max bracket size;
    **                           replaced Apache commons logging with J2SE logging.
    ** 030 ECF 20150929          Adjusted calls to modified BufferManager.evictDMOIfUnused method.
    ** 031 ECF 20160519          Increased bracket size progression base value.
    ** 032 ECF 20160610          Performance enhancement: process early brackets for projection
    **                           queries using a list operation instead of a scroll operation. The
    **                           former will add query results to the second-level query cache.
    ** 033 ECF 20160614          Reimplement #032 to not rely on projection queries and to report the
    **                           entities associated with the enclosing query when executing list or
    **                           scroll operations.
    ** 034 OM  20160720          Prevented ContextLocalCleanupException to be thrown when client
    **                           disconnects and the context is not accessible.
    ** 035 ECF 20160912          Persistence.list signature change.
    ** 036 ECF 20180709          Only use Persistence.list and SimpleResults for the first bracket of
    **                           full DMOs or if using a projection query. This prevents the session
    **                           cache from filling with DMOs not referenced by record buffers, which
    **                           can significantly slow session flush operations.
    ** 037 ECF 20180727          Raise an error condition when catching PersistenceException in
    **                           moveTo, rather than proceeding as normal.
    ** 038 ECF 20190821          Override default implementation of isResultSetCached.
    **     EVL 20190822          Adding NPE protection for isResultSetCached.
    ** 039 ECF 20200419          Removed Hibernate dependencies.
    ** 040 IAS 20220720          Fixed iteration logic (multiple iterations in different directions).
    **     SVL 20230113          Improved performance by replacing some "for-each" loops with indexed "for" loops.
    ** 041 IAS 20230116          Fixed FILL support for recursive DATA-RELATION
    ** 042 SR  20230510          Changed persistence.list() to match the new signature.
    ** 043 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
    ** 044 SR  20230728          Updated brackets to grow in size according to the memory allocated by records.
    **     AL2 20230801          Replaced constant with MIN_CHECKED_THRESHOLD.
    **     AB  20230821          Modified getResults(Locator locator) to not throw an error for an id null
    **                           (possible in cases of OUTER-JOIN).
    **     AB  20230822          Modified getResults(Locator locator) to use Objects.equals for comparing the
    **                           ids.
    ** 045 ASL 20231020          Added "lastValidBracket" variable to keep track of the most recent bracket value.
    **     ASL 20231024          Assign "lastValidBracket" to match the updates of "lastValidPosition".
    ** 046 DDF 20231220          Short-circuit moveTo when the Locator find the row in the same bracket
    **                           that was already searched.
    ** 047 OM  20231214          Persistence.list()'s last parameter forces the result to be hydrated (Record).
    ** 048 AL2 20240906          Implemented get(forceOnlyId).
    ** 049 DDF 20230825          Added ForwardResults.
    ** 050 LS  20241101          Modified getResults(Locator locator) to be able to compare ids even though
    **                           firstRow[i] is Record and lastRecord[i] is Long and vice versa.
    ** 051 OM  20241128          Multi-tenant runtime support: selected the proper persistence context, eventually
    **                           based on [sharedDb] parameter.
    ** 052 DDF 20241121          Added overload for fireResultsChanged() which allows the AdaptiveQuery to be
    **                           notified that the delegate was closed and allowing it to enable results
    **                           caching again.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist;

    import java.sql.*;
    import java.util.*;
    import java.util.logging.*;
    import com.goldencode.p2j.persist.event.*;
    import com.goldencode.p2j.util.ErrorManager;
    import com.goldencode.p2j.util.logging.*;

    /**
     * An implementation of the {@link Results} interface which is backed by
     * <code>Results</code> sets of progressively increasing size.
     * <p>
     * This results object is best used in queries which start at the beginning
     * of a result set and scroll forward sequentially.  While non-linear access
     * to records is possible, it is not the best choice for random access to
     * results.  In fact certain operations currently are disallowed, such as
     * testing whether the current position is the last available.  Thus, the
     * associated methods throw <code>UnsupportedOperationException</code> in the
     * current implementation.
     * <p>
     * This implementation is designed to balance the need for an immediate
     * initial result with the need for fast access to a large number of results.
     * <p>
     * To accomplish this, results are retrieved in brackets.  The initial fetch
     * retrieves a bracket containing a single record.  Each subsequent bracket
     * fetch retrieves a maximum size which is calculated by raising a base
     * number to a power one higher than that applied to the previous bracket.
     * So, for a base number of 10, the first bracket would have maximum 10^0
     * (or 1) record.  The second bracket (if any) would have maximum 10^1 (or
     * 10) records.  The third, 10^2 (or 100), and so on.  The default base value
     * is 10.
     * <p>
     * The exponential progression of bracket sizes ensures that the first
     * result is retrieved as quickly as possible, and subsequent brackets are
     * retrieved with reasonable efficiency.  Thus, queries which fall to the
     * smaller end of the spectrum will benefit from the quick retrieval of the
     * first few brackets, while the up front expense of retrieving larger
     * brackets will be amortized more evenly for those queries which process
     * larger data sets.
     * <p>
     * Note that currently, this progression is (effectively) not capped, which
     * presumes a reasonable JDBC fetch size to prevent a JDBC driver from caching
     * too many records and overrunning memory.
     * <p>
     * When considered end-to-end, in order of increasing size, all brackets
     * together form the virtualized set of results to a database query.  This
     * object keeps track of the virtual index position as it moves through the
     * various brackets.  Methods which accept an absolute row index expect this
     * value to represent the absolute offset of the target row in this overall,
     * virtualized result set, such that the bracketing implementation is an
     * implementation detail which is transparent to users of this class.
     * <p>
     * Only the most recently created result set bracket is retained.  Once all
     * records in a bracket have been retrieved and the walk proceeds to the next
     * record, the current bracket is discarded and a new one is retrieved.
     * Users of this class which require the ability to perform non-linear, non
     * first-to-last access must instantiate this class in scrolling mode.  In
     * this mode, a cache of visited records is maintained.  If a previously
     * accessed record must be re-retrieved, it is pulled from the cache.
     * Accordingly, scrolling instances may have a higher memory requirement.
     */
    final class ProgressiveResults
    implements Results
    {
       /** Logger */
       private static final CentralLogger LOG = CentralLogger.get(ProgressiveResults.class.getName());

       /** Base value used for exponential bracket size progression */
       private static final int BASE = 10;

       /**
        * Maximum exponent to which BASE can be raised in case we didn`t manage to analyze the optimal max
        * exponent based on row size.
        */
       private static final int FALLBACK_MAX_EXPONENT = 4;

       /** Maximum memory allocated for a bracket */
       private static final long MAX_RESULTS_SIZE = 1_000_000_000;

       /** The exponent used to get the average size of records */
       private static final int ANALYZE_EXPONENT = FALLBACK_MAX_EXPONENT - 1;

       /**
        * Minimum number of records that needs to be computed in order to set the maximum exponent.
        * This should be at most 2 ^ ANALYZE_EXPONENT.
        */
       private static final int MIN_CHECKED_THRESHOLD = 1000;

       /** Object which provides persistence services */
       private final Persistence persistence;

       /** HQL query string used to select records */
       private final String hql;

       /** Substitution parameters, if any, to be applied to the query */
       private final Object[] args;

       /** Cached results (scrolling mode only);  arrays of DMOs or IDs */
       private final List<Object[]> cache;

       /** Names of entities involved in the query associated with these results */
       private final Class<? extends Record>[] entities;

       /** Flag indicating whether these results are for a projection query */
       private final boolean projection;

       /** The recursive DATA-RELATION the query is to FILL for */
       private final DataRelation relation;

       /** Result set to which record retrieval work is delegated */
       private Results delegate = null;

       /** Current "real" position in the overall, virtualized result set */
       private int position = -1;

       /** Most recent valid "real" position in the virtualized result set */
       private int lastValidPosition = -1;

       /** Index of the active delegate */
       private int bracket = -1;

       /** Index of the most recent valid bracket */
       private int lastValidBracket = -1;

       /** List of results change listeners */
       private ArrayList<VirtualResultsListener> resultsChangeListeners;

       /** Flag indicates results are closed */
       private boolean closed = false;

       /** The sum of some records used to compute the size of the brackets */
       private long totalCheckedSize = 0;

       /** Numbers of records used to compute the size */
       private int checkedCount = 0;

       /** Actual maximum exponent in use */
       private Integer actualMaxExponent = null;

       /** Flag indicating the forward-only attribute value */
       private boolean forwardOnly = false;

       /**
        * The persistence context. Needed for notifying when this object is cleaned up so that the session can be
        * closed.
        */
       private final Persistence.Context[] persistenceContext = new Persistence.Context[2];

       /**
        * Convenience constructor which sets all variables required for record retrieval.
        *
        * @param   persistence
        *          Object which provides persistence services.
        * @param   hql
        *          HQL query string used to select records.
        * @param   args
        *          Substitution parameters, if any, to be applied to the query.
        * @param   scrolling
        *          {@code true} to allow results to be scrollable.  If {@code false}, it will only
        *          be possible to move forward through results, beginning at the first result;
        *          random access will be disallowed.
        * @param   forwardOnly
        *          Flag indicating the forward-only attribute value. If {@code true}, {@code ForwardResults}
        *          will be used instead of {@code ScrollingResults}.
        * @param   entities
        *          Names of entities associated with these results.
        * @param   projection
        *          Flag indicating whether these results are for a projection query.
        * @param   relation
        *          The recursive DATA-RELATION the query is to FILL for.
        */
       ProgressiveResults(Persistence persistence,
                          String hql,
                          Object[] args,
                          boolean scrolling,
                          boolean forwardOnly,
                          Class<? extends Record>[] entities,
                          boolean projection,
                          DataRelation relation)
       {
          this.persistence = persistence;
          this.hql = hql;
          this.args = args;
          this.cache = (scrolling ? new ArrayList<>() : null);
          this.forwardOnly = forwardOnly;
          this.entities = entities;
          this.projection = projection;
          this.persistenceContext[0] = persistence.getContext(true);
          this.persistenceContext[1] = persistence.getContext(false);
          this.relation = relation;

          persistenceContext[0].useSession();
          if (persistenceContext[0] != persistenceContext[1])
          {
             persistenceContext[1].useSession();
          }
       }

       /**
        * Move to the first result row.  If a previous move has taken us beyond
        * the first row, we must be in scrolling mode to be able to go back.
        *
        * @return  <code>true</code> if there are any results.
        *
        * @throws  UnsupportedOperationException
        *          if moving to the first record would scroll backwards, and this
        *          object is not set to scrolling mode.
        */
       public boolean first()
       {
          closeDelegate();
          position = -1;
          lastValidPosition = -1;
          lastValidBracket = -1;
          bracket = -1;
          return moveTo(0);
       }

       /**
        * Move to the last result row.  This can be a very expensive operation
        * for a large result set, as we must scroll through each record to reach
        * the end.  If we are already at the final record, this method does not
        * change the position, but does report success.
        *
        * @return  <code>true</code> if there are any results.
        */
       public boolean last()
       {
          boolean valid = (position >= 0);

          // Advance until we can go no further.  This will take us off end.
          while (next())
          {
             valid = true;
          }

          // If we were able to advance at all, assume we were able to advance
          // off end.  Now reset position to its highest valid value to get us
          // back to the last record.  This also takes into account the case in
          // which we were already positioned at the last valid record coming
          // into this method.
          if (valid)
          {
             position = lastValidPosition;
             bracket = lastValidBracket;
             if (cache == null)
             {
                moveTo(position, true);
             }
          }

          return valid;
       }

       /**
        * Move cursor to the next result row.
        *
        * @return  <code>true</code> if there is a result under the cursor
        *          after the move.
        */
       public boolean next()
       {
          return moveTo(position + 1);
       }

       /**
        * Move cursor to the previous result row.
        *
        * @return  <code>true</code> if there is a result under the cursor
        *          after the move.
        */
       public boolean previous()
       {
          return bracket == -1 ? last() : moveTo(position - 1);
       }

       /**
        * Is the cursor on the first row in the result set?
        *
        * @return  <code>true</code> if the cursor is on the first row.
        */
       public boolean isFirst()
       {
          return (position == 0);
       }

       /**
        * Indicate whether the current record is the last row in the overall,
        * virtualized result set.
        * <p>
        * This method is not supported by this implementation.
        *
        * @return  <code>true</code> if the cursor is on the last row, else
        *          <code>false</code>.
        *
        * @throws  UnsupportedOperationException
        *          always.
        */
       public boolean isLast()
       {
          throw new UnsupportedOperationException();
       }

       /**
        * Should get the primary keys array for the objects at the current result row when implemented by
        * subclass and forceOnlyId is true. Otherwise it returns the array with the whole objects at the
        * current result row (which can be either PK or actual DMOs).
        *
        * @param   forceOnlyId
        *          {@code true} if this requires only the ID, not the full record.
        *
        * @return  Object array or {@code null}.
        */
       public Object[] get(boolean forceOnlyId)
       {
          if (position < 0)
          {
             return null;
          }

          Object[] result;

          // If we have a cache and it contains the requested record, retrieve from the cache.
          if (cache != null && position < cache.size())
          {
             result = cache.get(position);
             if (forceOnlyId)
             {
                // make a new array to avoid altering the cache
                Object[] results = new Object[result.length];
                for (int i = 0; i < result.length; i++)
                {
                   if (result[i] instanceof Record)
                   {
                      results[i] = ((Record) result[i]).primaryKey();
                   }
                   else
                   {
                      results[i] = result[i];
                   }
                }
                return results;
             }
             else
             {
                analyzeRow(result);
                return result;
             }
          }

          try
          {
             // Return datum from current results, if any.
             Results results = getResults(bracket);
             if (results != null)
             {
                result = results.get(forceOnlyId);
                if (!forceOnlyId)
                {
                   analyzeRow(result);
                }
                return result;
             }
          }
          catch (Exception exc)
          {
             cleanup();

             if (LOG.isLoggable(Level.FINE))
             {
                LOG.log(Level.FINE, exc.getMessage(), exc);
             }
             else if (LOG.isLoggable(Level.WARNING))
             {
                LOG.log(Level.WARNING, exc.getMessage());
             }

             DBUtils.handleException(persistence.getDatabase(Persistence.PRIVATE_CTX), exc);
          }

          return null;
       }

       /**
        * Get the object at the current result row, at the specified column.
        *
        * @param   column
        *          Zero-based index column of the desired object.
        *
        * @return  Object at <code>column</code> or <code>null</code>.
        */
       public Object get(int column)
       {
          Object[] row = get();

          return (row != null ? row[column] : null);
       }

       /**
        * Indicate whether the full result set is cached in this object.
        *
        * @return  {@code true} if all results are contained this object; {@code false} if not or
        *          if unknown (such as for a scrolling or cursored result set. Default return value
        *          is {@code false}.
        */
       @Override
       public boolean isResultSetCached()
       {
          return delegate == null ? false : delegate.isResultSetCached();
       }

       /**
        * Get the primary key ID at the current result row, at the specified
        * column.
        *
        * @param   column
        *          Zero-based index column of the desired ID.
        *
        * @return  ID of the record or <code>null</code> if there is no record at
        *          the current position.
        */
       public Long getID(int column)
       {
          Object result = get(column);
          if (result instanceof Record)
          {
             return ((Record) result).primaryKey();
          }

          return (Long) result;
       }

       /**
        * Get the row number currently under the cursor.
        *
        * @return  Zero-based index of the current row, or <code>-1</code>
        *          if the cursor is not currently on a result.
        */
       public int getRowNumber()
       {
          return position;
       }

       /**
        * Set the row number currently under the cursor.
        *
        * @param   row
        *          Zero-based index of the row to be set as the current row.
        *
        * @return  <code>true</code> if there is a row at the specified row
        *          number;  else <code>false</code>.
        */
       public boolean setRowNumber(int row)
       {
          return moveTo(row);
       }

       /**
        * Scroll the cursor ahead by the specified number of rows.  If the
        * number is negative, the cursor is moved backward.  This may put the
        * cursor off the end (either end) of the query's results.
        *
        * @param   rows
        *          Number of rows to jump ahead or back.
        *
        * @return  <code>true</code> if there is a row at the new location;
        *          else <code>false</code>.
        */
       public boolean scroll(int rows)
       {
          return moveTo(position + rows);
       }

       /**
        * Reset the cursor to its natural starting position, before the first
        * result row.  Only valid for scrolling queries.
        *
        * @throws  UnsupportedOperationException
        *          if invoked on a non-scrolling results instance.
        */
       public void reset()
       {
          if (cache == null && bracket > 0)
          {
             throw new UnsupportedOperationException(
                "Unable to reset non-scrolling query results");
          }

          position = -1;
       }

       /**
        * Respond to a session closing event by caching a batch of this results
        * object's records. Only the current record and all remaining records
        * within the current bracket are cached.  If no records have yet been
        * retrieved, or if the last record retrieved was the last in its bracket,
        * then no records are cached.
        */
       public void sessionClosing()
       {
          if (closed                ||
              bracket < 0           ||
              position < 0          ||
              lastValidPosition != position ||
              delegate == null      ||
              delegate instanceof SimpleResults)
          {
             return;
          }

          Locator locator = locateRow(position);
          int size = locator.getSize() - locator.getOffset();
          ArrayList<Object[]> rows = new ArrayList<>(size);

          try
          {
             do
             {
                rows.add(delegate.get());
             } while (delegate.next());
          }
          finally
          {
             closeDelegate();
          }
          rows.trimToSize();

          delegate = new SimpleResults(rows);
          delegate.setRowNumber(0);
       }

       /**
        * Close an open result set, if any, and clear the cache, if any.  Reset state variables to their defaults.
        */
       public void cleanup()
       {
          if (closed)
          {
             return;
          }

          try
          {
             closeDelegate();
             position = -1;
             lastValidPosition = -1;
             lastValidBracket = -1;
             bracket = -1;

             if (cache != null)
             {
                cache.clear();
             }
          }
          finally
          {
             closed = true;
             persistenceContext[0].releaseSession();
             if (persistenceContext[0] != persistenceContext[1])
             {
                persistenceContext[1].releaseSession();
             }
          }
       }

       /**
        * Register results change listener. This listener will be notified about
        * changes in obtained results.
        *
        * @param listener
        *        Listener to add.
        */
       public void addResultsChangeListener(VirtualResultsListener listener)
       {
          if (resultsChangeListeners == null)
          {
             resultsChangeListeners = new ArrayList<>();
          }

          resultsChangeListeners.add(listener);
       }

       /**
        * Generate a descriptor object which is used to locate the bracket
        * containing the specified row and the row's location within the bracket.
        * <p>
        * The object returned does not reflect the state of records or whether
        * the target record actually exists in the result set.  Rather, this
        * method calculates where the record should be located based upon the
        * bracketing algorithm in use.
        *
        * @param   row
        *          0-based index of the target record in the virtualized result
        *          set (i.e., the record's "real" position among the full result
        *          set of the query, ignoring bracketing).
        *
        * @return  A descriptor object which allows the target record to be
        *          located within the appropriate bracket of results.
        */
       private Locator locateRow(int row)
       {
          // Determine bracket and offset within bracket.
          int bracket;
          int test = 0;
          int previous = 0;
          int size = 0;
          int cumulative = 0;

          for (bracket = 0; test <= row; bracket++)
          {
             previous = test;
             cumulative += size;

             if (bracket < FALLBACK_MAX_EXPONENT)
             {
                size = (int) Math.pow(BASE, bracket);
             }
             else
             {
                size = (int) Math.pow(BASE,Math.min(bracket, getMaxExponent()));
             }
             test = size + previous;
          }
          bracket--;
          int offset = row - cumulative;

          Locator locator = new Locator(bracket, offset, size, row);

          if (LOG.isLoggable(Level.FINEST))
          {
             LOG.log(Level.FINEST, "locateRow(" + row + ")..." + locator);
          }

          return locator;
       }

       /**
        * Generate a descriptor object which is used to locate the bracket
        * at the given index among all results brackets.
        * <p>
        * The object returned does not reflect the state of records or whether
        * the target bracket actually exists.  Rather, this method calculates
        * the parameters required to request the records which would comprise
        * the bracket, according to the bracketing algorithm in use.
        * <p>
        * The <code>Locator</code> object returned will contain the specified
        * bracket, an offset of 0, the size of the bracket, and its row will
        * reflect the zero-based offset (in the virtualized result set) of the
        * row which starts the target bracket.
        *
        * @param   bracket
        *          0-based index of the target bracket in the virtualized result
        *          set.
        *
        * @return  A descriptor object which allows the target bracket to be
        *          created.
        */
       private Locator locateBracket(int bracket)
       {
          int startRow = 0;
          int size = 0;
          int i;
          for (i = 0; i <= bracket; i++)
          {
             if(i < FALLBACK_MAX_EXPONENT)
             {
                size = (int) Math.pow(BASE, i);
             }
             else
             {
                size = (int) Math.pow(BASE, Math.min(i, getMaxExponent()));
             }

             if (i < bracket)
             {
                startRow += size;
             }
          }

          return (new Locator(bracket, 0, size, startRow));
       }

       /**
        * Attempt to move to a specific row offset within the virtualized result
        * set, and report whether a record exists at that row.
        * <p>
        * This method updates the internal state variables which track the
        * current virtualized row index and bracket index.  Both are set to their
        * default values if the target row does not exist.
        *
        * @param   row
        *          0-based index of the target record in the virtualized result
        *          set (i.e., the record's "real" position among the full result
        *          set of the query, ignoring bracketing).
        *
        * @return  <code>true</code> if there is a row at the specified row
        *          number;  else <code>false</code>.
        *
        * @throws  UnsupportedOperationException
        *          if a backwards move is requested for a non-scrolling results
        *          object.
        */
       private boolean moveTo(int row)
       {
          return moveTo(row, false);
       }

       /**
        * Attempt to move to a specific row offset within the virtualized result
        * set, and report whether a record exists at that row.
        * <p>
        * This method updates the internal state variables which track the
        * current virtualized row index and bracket index.  Both are set to their
        * default values if the target row does not exist.
        *
        * @param   row
        *          0-based index of the target record in the virtualized result
        *          set (i.e., the record's "real" position among the full result
        *          set of the query, ignoring bracketing).
        * @param   force
        *          Force move even if the target row matches the current row.
        *          Allows to initialize delegate in some cases.
        *
        * @return  <code>true</code> if there is a row at the specified row
        *          number;  else <code>false</code>.
        *
        * @throws  UnsupportedOperationException
        *          if a backwards move is requested for a non-scrolling results
        *          object.
        */
       private boolean moveTo(int row, boolean force)
       {
    //      // Ensure we have a cache if backward scrolling is needed.
    //      if (row < position && cache == null)
    //      {
    //         if (row != 0)
    //         {
    //            throw new UnsupportedOperationException(
    //               "Backward scroll not permitted for non-scrolling query");
    //         }
    //      }

          // If requested row is negative, move before the first record.
          if (row < 0)
          {
             lastValidPosition = (position >= 0 ? 0 : -1);
             lastValidBracket = this.bracket;
             position = -1;

             return false;
          }

          // No move necessary if we are already positioned at the requested row.
          if (!force && row == position && row == lastValidPosition)
          {
             return true;
          }

          // Use the cache if possible.
          if (cache != null)
          {
             int size = cache.size();

             if (row < size)
             {
                position = row;
                lastValidPosition = position;
                lastValidBracket = this.bracket;

                return true;
             }
             else
             {
                position = size - 1;
             }
          }

          // From this point on, it is safe to assume the move is forward.  If
          // we have a cache, scrolling forward must add entries to it.

          boolean valid = false;

          try
          {
             // Try to move within the current bracket first to see if it contains
             // the desired row.  This is less expensive than creating a locator
             // object.
             Results results = getResults(bracket);
             if (results != null)
             {
                if (cache != null)
                {
                   valid = advanceCache(results, row);
                   if (position == row && valid)
                   {
                      lastValidPosition = position;
                      lastValidBracket = bracket;

                      return true;
                   }
                }
                else
                {
                   int delta = row - Math.max(0, position);
                   valid = results.scroll(delta);
                   if (valid)
                   {
                      position = row;
                      lastValidPosition = position;
                      lastValidBracket = bracket;

                      return true;
                   }
                }
             }

             // The current bracket did not contain the target row.  Use a locator
             // to get the proper bracket and move to the target row within it.
             Locator locator = locateRow(row);

             if (results != null && locator.getBracket() == this.bracket)
             {
                return false;
             }

             // If we are caching, retrieve every result set and cache results.
             if (cache != null)
             {
                int bracket = locator.getBracket();
                while (this.bracket < bracket)
                {
                   results = getResults(this.bracket + 1);

                   if (results == null)
                   {
                      break;
                   }

                   valid = advanceCache(results, row);
                }

                lastValidPosition = position;
                lastValidBracket = this.bracket;

                if (position == row && valid)
                {
                   return true;
                }
                else
                {
                   position = (lastValidPosition >= 0 && row >= lastValidPosition ? lastValidPosition + 1 : -1);

                   return false;
                }
             }

             results = getResults(locator);

             if (results == null)
             {
                // Move was not possible.  Reset position to default and give up.
                if (position >= 0)
                {
                   lastValidPosition = position;
                   lastValidBracket = this.bracket;
                }
                position = (lastValidPosition >= 0 && row >= lastValidPosition ? lastValidPosition + 1 : -1);

                return false;
             }

             // We found a bracket which might contain our target row;  try to move to it.

             // if we have retrieved an extra first row for comparing with the
             // last row of the previous bracket then actual row number will
             // be locator.offset + 1
             int rowNumber = locator.getRow() > 0 ? locator.getOffset() + 1 : locator.getOffset();
             valid = results.setRowNumber(rowNumber);
             if (valid)
             {
                // Update state to reflect move.
                position = row;
                lastValidPosition = position;
                bracket = locator.getBracket();
                lastValidBracket = bracket;
             }
          }
          catch (PersistenceException exc)
          {
             cleanup();

             if (LOG.isLoggable(Level.FINE))
             {
                LOG.log(Level.FINE, exc.getMessage(), exc);
             }
             else if (LOG.isLoggable(Level.WARNING))
             {
                LOG.log(Level.WARNING, exc.getMessage());
             }

             valid = false;

             DBUtils.handleException(persistence.getDatabase(Persistence.PRIVATE_CTX), exc);

             // at this point, we have a real and unexpected error and we can't go on, because
             // calling code will just treat this like a normal off-end condition
             ErrorManager.throwError(exc);
          }
          finally
          {
             // Eagerly close the underlying result set if there was no valid
             // result.
             if (!valid)
             {
                closeDelegate();
             }
          }

          return valid;
       }

       /**
        * Sequentially retrieve records and store them in the cache until either
        * there are no more records, or until the current position has reached
        * <code>row</code>.
        *
        * @param   results
        *          Delegate results object from which to retrieve records.
        * @param   row
        *          Target row index.  Retrieval halts when the current record
        *          position reaches this row.
        *
        * @return  <code>true</code> if the last attempt to retrieve a record was
        *          successful;  <code>false</code> if we reached the end of the
        *          delegate result set bracket before reaching our target row.
        *
        * @throws  IllegalStateException
        *          if the current position is not at the last record in the
        *          cache.
        */
       private boolean advanceCache(Results results, int row)
       {
          // Sanity check.
          if (position + 1 != cache.size())
          {
             throw new IllegalStateException("Progressive results cache is corrupt");
          }

          boolean valid = false;

          while (position < row && (valid = results.next()))
          {
             position++;
             cache.add(results.get());
          }

          return valid;
       }

       /**
        * Access the delegate result set object for the given bracket index.  If
        * the bracket does not yet exist, it is created if possible.
        *
        * @param   bracket
        *          0-based index of the target bracket.
        *
        * @return  Delegate <code>Results</code> for the given bracket,
        *          or <code>null</code> if no such bracket exists.
        */
       private Results getResults(int bracket)
       throws PersistenceException
       {
          if (bracket == this.bracket)
          {
             return delegate;
          }
          else if (bracket < this.bracket)
          {
             return null;
          }

          Locator locator = locateBracket(bracket);

          return getResults(locator);
       }

       /**
        * Access the delegate result set object defined by the given locator.
        * If the bracket does not yet exist, create it if possible.
        *
        * @param   locator
        *          Descriptor of the target bracket.
        *
        * @return  Delegate <code>Results</code> for the given bracket,
        *          or <code>null</code> if no such bracket exists.
        *
        * @throws  PersistenceException
        *          if there is an error retrieving a result set.
        */
       private Results getResults(Locator locator)
       throws PersistenceException
       {
          int bracket = locator.getBracket();

          if (bracket > this.bracket || (bracket == this.bracket && delegate == null))
          {
             // Save the last record of the previous bracket.
             Object[] lastRecord = null;
             if (delegate != null && delegate.last())
             {
                lastRecord = delegate.get();
             }

             closeDelegate();

             // we should retrieve extra record at the front end of the results set if it is not the
             // first bracket
             int start      = locator.getRow() - locator.getOffset();
             int offset     = start > 0 ? start - 1 : start;
             int maxResults = start > 0 ? locator.getSize() + 1 : locator.getSize();

             if (bracket == 0 || (bracket <= 3 && projection))
             {
                // process the early, smaller brackets using a list query, so the ORM adds the results to the
                // second level query cache; only do this for projection queries, otherwise we end up with
                // unwanted entities in the session cache, which makes the dirty checking cycle unduly expensive
                delegate = new SimpleResults(
                      persistence.list(entities, hql, args, maxResults, offset, relation, true));
             }
             else if (forwardOnly)
             {
                delegate = new ForwardResults(persistence,
                                              persistence.scroll(entities,
                                                                 hql,
                                                                 args,
                                                                 maxResults,
                                                                 offset,
                                                                 relation,
                                                                 ResultSet.TYPE_FORWARD_ONLY));
             }
             else
             {
                // process later, larger brackets or those of non-projection queries as scrollable
                // results; these will not be added to the query cache
                delegate =
                   new ScrollingResults(persistence,
                                        persistence.scroll(entities,
                                                           hql,
                                                           args,
                                                           maxResults,
                                                           offset,
                                                           relation,
                                                           ResultSet.TYPE_SCROLL_INSENSITIVE));
             }

             // compare the last record of the previous bracket and the first bracket of the new
             // record
             if (lastRecord != null && delegate.first())
             {
                Object[] firstRow = delegate.get();
                int frLen;
                if (firstRow != null && (frLen = firstRow.length) == lastRecord.length)
                {

                   for (int i = 0; i < frLen; i++)
                   {
                      Long firstId = firstRow[i] instanceof Record ?
                              ((Record) firstRow[i]).primaryKey() : (Long) firstRow[i];
                      Long lastId = lastRecord[i] instanceof Record ?
                              ((Record) lastRecord[i]).primaryKey() : (Long) lastRecord[i];

                      if (!Objects.equals(firstId, lastId))
                      {
                         fireResultsChanged();
                         return null;
                      }
                   }
                }
             }

             if (lastRecord != null)
             {
                // evict lastRecord DMOs if this is the only use which would pin them in the ORM session
                BufferManager bufMgr = null;
                for (Object next : lastRecord)
                {
                   if (next instanceof Record)
                   {
                      if (bufMgr == null)
                      {
                         bufMgr = BufferManager.get();
                      }
                      Record dmo = (Record) next;
                      bufMgr.evictDMOIfUnused(persistence, null, dmo);
                   }
                }
             }

             this.bracket = bracket;
          }

          return delegate;
       }

       /**
        * Close the delegate result set, if any.
        */
       private void closeDelegate()
       {
          if (delegate != null)
          {
             try
             {
                delegate.cleanup();
                fireResultsChanged(true);
             }
             catch (Exception exc)
             {
                // Non-critical operation;  logging the error is sufficient.
                if (LOG.isLoggable(Level.WARNING))
                {
                   if (LOG.isLoggable(Level.FINE))
                   {
                      LOG.log(Level.FINE, exc.getMessage(), exc);
                   }
                   else
                   {
                      LOG.log(Level.WARNING, exc.getMessage());
                   }
                }

                DBUtils.handleException(persistence.getDatabase(Persistence.PRIVATE_CTX), exc);
             }
             finally
             {
                delegate = null;
             }
          }
       }

       /**
        * Notify all listeners about changes in obtained results.
        */
       private void fireResultsChanged()
       {
          fireResultsChanged(false);
       }

       /**
        * Notify all listeners about changes in obtained results.
        * This method notifies the listener when the delegate results have changed or when the
        * actual delegate is replaced by a new one (entering a new bracket), this allows the
        * query to cache the results again.
        *
        * @param   delegateChanged
        *          {@code true} when the delegate is closed, {@code false} otherwise.
        */
       private void fireResultsChanged(boolean delegateChanged)
       {
          if (resultsChangeListeners != null)
          {
             for (int i = 0; i < resultsChangeListeners.size(); i++)
             {
                VirtualResultsListener listener = resultsChangeListeners.get(i);
                if (!delegateChanged)
                {
                   listener.resultsChanged();
                }
                else
                {
                   listener.delegateChanged();
                }
             }
          }
       }

       /**
        * Calculate the largest exponent in regard to the row size analysis we done to determine the
        * optimal size for the brackets. This is a heuristic, so it won't guarantee that the brackets won't
        * exceed {@link #MAX_RESULTS_SIZE}. However, this is an effort to avoid retrieving a lot of records
        * at once to trigger OOM.
        *
        * @return   The maximum exponent that should be used in regard to the row size analysis done.
        */
       private int getMaxExponent()
       {
          if (actualMaxExponent != null)
          {
             return actualMaxExponent;
          }
          if (checkedCount < MIN_CHECKED_THRESHOLD || totalCheckedSize == 0)
          {
             return FALLBACK_MAX_EXPONENT;
          }

          long nrOfRows = (long) (MAX_RESULTS_SIZE / (totalCheckedSize / checkedCount));
          int exp = 0;
          long result = 1;
          while (result < nrOfRows)
          {
             exp++;
             result *= BASE;
          }
          actualMaxExponent = exp - 1;

          if (actualMaxExponent < FALLBACK_MAX_EXPONENT)
          {
             actualMaxExponent = FALLBACK_MAX_EXPONENT;
          }

          return actualMaxExponent;
       }

       /**
        * Calculates the size of a row in bytes.
        *
        * @param   result
        *          The row that should be analyzed.
        */
       private void analyzeRow(Object[] result)
       {
          if (actualMaxExponent == null && bracket == ANALYZE_EXPONENT && checkedCount < MIN_CHECKED_THRESHOLD)
          {
             for (int i = 0; i < result.length; i++)
             {
                if (result[i] instanceof Record)
                {
                   totalCheckedSize += ((Record) result[i]).getSize();
                }
                else
                {
                   // result[i] can be Long so we are adding 4 bytes for now
                   totalCheckedSize += 4;
                }
             }
             checkedCount++;
          }
       }

       /**
        * A descriptor object used to locate results brackets and store
        * information used to access them.  The parameters stored in this object
        * are used both to identify a particular row among the brackets of
        * results, and to create new brackets.
        */
       private static class Locator
       {
          /** 0-based bracket index within list of delegate result set objects */
          private int bracket = -1;

          /** Offset of a particular record from the start of a bracket */
          private int offset = -1;

          /** Number of records in a bracket */
          private int size = 0;

          /** 0-based index of a row in the virtualized result set */
          private int row = 0;

          /**
           * Convenience constructor.
           *
           * @param   bracket
           *          0-based bracket index within a list of delegate result set
           *          objects.
           * @param   offset
           *          Offset of a particular record from the start of a bracket.
           * @param   size
           *          Number of records in a bracket.
           * @param   row
           *          0-based index of a row in the virtualized result set.
           */
          Locator(int bracket, int offset, int size, int row)
          {
             this.bracket = bracket;
             this.offset = offset;
             this.size = size;
             this.row = row;
          }

          /**
           * Get the 0-based bracket index within a list of delegate result set
           * objects.
           *
           * @return  Bracket index.
           */
          int getBracket()
          {
             return bracket;
          }

          /**
           * Get the offset of a particular record from the start of the bracket.
           *
           * @return  Row offset within a bracket.
           */
          int getOffset()
          {
             return offset;
          }

          /**
           * Get the number of records in the bracket.
           *
           * @return  Bracket size.
           */
          int getSize()
          {
             return size;
          }

          /**
           * Get the 0-based index of a row in the virtualized result set.
           *
           * @return  Offset of row from the start of all brackets.
           */
          int getRow()
          {
             return row;
          }

          /**
           * Get a string reprentation of this object's current state.
           *
           * @return  String which describes this locator.
           */
          public String toString()
          {
             return "[bracket: " + bracket +
                    ", offset: " + offset +
                    ", size: " + size +
                    ", row: " + row + "]";
          }
       }
    }