# MultiSessionAppserver.java

    /*
    ** Module   : MultiSessionAppserver.java
    ** Abstract : Appserver with multi-session agents.
    **
    ** Copyright (c) 2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------Description----------------------------------
    ** 001 GBB 20250122 Created initial version.
    */
    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util.appserver;

    import com.goldencode.p2j.admin.*;
    import com.goldencode.p2j.main.*;
    import com.goldencode.p2j.oo.lang.*;
    import com.goldencode.p2j.security.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.p2j.util.logging.*;

    import javax.annotation.*;
    import java.lang.ref.*;
    import java.util.*;
    import java.util.concurrent.*;
    import java.util.concurrent.atomic.*;
    import java.util.function.*;
    import java.util.logging.*;

    /** Appserver with multi-session agents. */
    class MultiSessionAppserver
    {
       /** Enum for the state of the multi-session appserver */
       enum State
       {
          INITIALIZING,
          RUNNING,
          TERMINATING,
          TERMINATED
       }

       /** ID of invalid id, used to indicate no session found. */
       static final int INVALID_SESSION_ID = -1;

       /** Logger. */
       private static final CentralLogger LOG = CentralLogger.get(MultiSessionAppserver.class);

       /** The Future for reserving a session, when the result is an invalid session. */
       private static final CompletableFuture<Integer> NO_SESSION_FOUND_RESULT =
          CompletableFuture.completedFuture(INVALID_SESSION_ID);

       /** The name prefix for the worker threads. */
       private static final String WORKER_THREAD_NAME_PREFIX = "MSA Worker #";

       /** The session global scope */
       private static final String SESSION_GLOBAL_SCOPE = "msa-session";

       /**
        * Undocumented OE config 'minAvailableABLSessions', min number of MSA sessions is always equal or bigger
        * than 1.
        */
       private static final int MIN_ABL_SESSIONS = 1;

       /** Min number of worker threads at any time. */
       private static final int MIN_CONNECTIONS = 1;

       /** ID generator for the MSA tasks. */
       private static final AtomicInteger msaTaskIdGenerator = new AtomicInteger(1);

       /** The definition of the current appserver. */
       private final MultiSessionAppserverDefinition definition;

       /** The agent connection for the system tasks. */
       private final MultiSessionClientConnection agentAdminConnection;

       /** Counter for the incremental id of the worker threads. */
       private final AtomicInteger workerNumberCounter = new AtomicInteger(1);

       /** The idle resources watchdog. Kept as a field to prevent being GCed. */
       private final ScheduledExecutorService watchdogExecutor =
          Executors.newSingleThreadScheduledExecutor(r ->
          {
             Thread newThread = new Thread(r, "MSA Idle Resources Watchdog");
             newThread.setDaemon(true);
             return newThread;
          });

       /** The startup time for the current appserver. */
       private final Date startupTime;

       /** Sorted set of available unbound sessions in idle state. */
       private final TreeSet<Integer> idleSessions = new TreeSet<>();

       /** Sorted set of available bound sessions in idle state. */
       private final Set<Integer> idleBoundSessions = new HashSet<>();

       /** Reserved session id : connection id pairs. */
       private final Map<Integer, String> connIdByReservedSessionId = new HashMap<>();

       /** Bound session id : connection id pairs. Bound connections can be idle/reserved/active. */
       private final Map<Integer, String> connIdByBoundSessionId = new HashMap<>();

       /** Active session id : connection id pairs. */
       private final Map<Integer, String> connIdByActiveSessionId = new HashMap<>();

       /** Map of session ids and their creation dates. */
       private final Map<Integer, Date> sessionCreateTime = new ConcurrentHashMap<>();

       /** List of listeners waiting for a free unbound session. */
       private final List<SessionListenerFuture> unboundSessionListeners = new ArrayList<>();

       /** List of listeners waiting for a free bound session. */
       private final List<SessionListenerFuture> boundSessionListeners = new ArrayList<>();

       /** The lock to synchronize any agent session operations */
       private final Object sessionsLock = new Object();

       /** The security manager for the multi-session agent appservers. */
       private final MultiSessionAppserverSecurityManager msaSecurityManager =
          MultiSessionAppserverSecurityManager.getInstance();

       /** LOG-MANAGER instance used for logging appserver logs out of appserver context. */
       private LegacyLogManager outOfContextLogManager;

       /** The startup parameters for the agent. */
       private StartupParameters startupParameters;

       /** The executor for the worker thread pool. */
       private ThreadPoolExecutor workerPoolExecutor;

       /** The queued tasks to be running in the {@link #workerPoolExecutor}. */
       private LinkedBlockingQueue<Runnable> taskQueue;

       /** Listener for freed sessions, receiving calls during agent termination. */
       private TerminationSessionListener terminationSessionListener;

       /** The EXPORTS for this appserver. */
       private volatile String exports;

       /** The current state of the appserver. */
       private volatile State state;

       /**
        * Package-private constructor.
        *
        * @param    definition
        *           The definition for the appserver to be started.
        * @param    connectionsSupplier
        *           Supplier for all currently active connections.
        * @param    disconnectMultipleFunc
        *           Function to disconnect connections, provided in a collection.
        *
        * @throws   Exception
        *           Any exception thrown during starting the appserver.
        */
       MultiSessionAppserver(MultiSessionAppserverDefinition definition,
                             Supplier<Collection<MultiSessionClientConnection>> connectionsSupplier,
                             Consumer<Collection<MultiSessionClientConnection>> disconnectMultipleFunc)
       throws Exception
       {
          this.definition = definition;
          this.state = State.INITIALIZING;
          this.agentAdminConnection = new MultiSessionClientConnection(MultiSessionClientConnection.AGENT_SYS_CONN_ID,
                                                                       null,
                                                                       definition.getAppserverName(),
                                                                       false,
                                                                       false);

          prepareLogging();

          AgentTerminationTask terminationTask = new AgentTerminationTask(MultiSessionAgent.SINGLE_AGENT_ID,
                                                                          definition.getCompleteActiveReqTimeout(),
                                                                          MultiSessionAgent.ExitCode.EXPECTED_4020,
                                                                          true);
          OrderedShutdownHooks.registerShutdownHook(1, terminationTask);

          initializeAgent();

          this.state = State.RUNNING;
          this.startupTime = new Date();

          int watchdogPeriod = definition.getIdleResourceTimeout();
          int clientConnectionTimeout = definition.getIdleSessionTimeout();
          if (watchdogPeriod > 0 && clientConnectionTimeout > 0)
          {
             watchdogExecutor.scheduleAtFixedRate(new Runnable()
             {
                String appserverName = definition.getAppserverName();

                @Override
                public void run()
                {
                   if (LOG.isLoggable(Level.FINEST))
                   {
                      LOG.finest("Running MSA idle resources watchdog.");
                   }

                   long now = new Date().getTime();

                   // mark client connections for removal
                   List<MultiSessionClientConnection> connectionsToDisconnect = new ArrayList<>();
                   for (MultiSessionClientConnection connection : connectionsSupplier.get())
                   {
                      if (connection.getAppserverName().equals(appserverName) &&
                         !connection.getId().equals(MultiSessionClientConnection.AGENT_SYS_CONN_ID) &&
                         !connection.isForWebService() &&
                         (connection.getParentConnection() == null ||
                            !connection.getParentConnection().isForWebService()) &&
                         !connection.isInUse() &&
                         !connection.isDisconnecting() &&
                         connection.getLastUseTime().getTime() + clientConnectionTimeout < now)
                      {
                         connectionsToDisconnect.add(connection);
                      }
                   }

                   if (connectionsToDisconnect.size() > 0 && LOG.isLoggable(Level.FINE))
                   {
                      LOG.fine("Idle connections: " + connectionsToDisconnect.size());
                   }

                   disconnectMultipleFunc.accept(connectionsToDisconnect);
                }
             }, watchdogPeriod, watchdogPeriod, TimeUnit.MILLISECONDS);
          }
       }

       /**
        * Returns the definition for the appserver.
        *
        * @return   See above.
        */
       MultiSessionAppserverDefinition getDef()
       {
          return definition;
       }

       /**
        * Returns the number of running agents.
        *
        * @return   See above.
        */
       int getNumberOfAgents()
       {
          return 1;
       }

       /**
        * Returns the number of the started sessions (idle, reserved and active).
        *
        * @return   See above.
        */
       int getNumberOfSessions()
       {
          synchronized (sessionsLock)
          {
             return idleSessions.size() + idleBoundSessions.size() +
                connIdByReservedSessionId.size() + connIdByActiveSessionId.size();
          }
       }

       /**
        * Returns the time of the appserver start.
        *
        * @return   See above.
        */
       Date getStartupTime()
       {
          return startupTime;
       }

       /**
        * Returns the client connection used for system tasks.
        *
        * @return   See above.
        */
       MultiSessionClientConnection getAdminConnection()
       {
          return agentAdminConnection;
       }

       /**
        * Get the exports for this appserver.  If set, only the external procedures which match these
        * exports will be available for invocation.
        *
        * @return   See above.
        */
       character getExports()
       {
          return exports == null ? null : new character(exports);
       }

       /**
        * Set the exports for this appserver.
        *
        * @param    exports
        *           The appserver's exports.
        *
        * @return   <code>true</code> if the method succeeded.
        */
       logical setExports(character exports)
       {
          this.exports = (exports.isUnknown() ? null : exports.toStringMessage());

          return new logical(true);
       }

       /**
        * Returns an array of descriptions for all started sessions (idle, reserved and active).
        *
        * @return   See above.
        */
       MultiSessionAgentSessionDef[] getSessionDefs()
       {
          List<MultiSessionAgentSessionDef> sessionInfos = new ArrayList<>();
          String appserverName = definition.getAppserverName();
          MultiSessionAppserverManager msaManager = MultiSessionAppserverManager.getInstance();

          synchronized (sessionsLock)
          {
             for (int idleSessionId : idleSessions)
             {
                sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                                                 idleSessionId,
                                                                 sessionCreateTime.get(idleSessionId),
                                                                 false,
                                                                 MultiSessionAgentSessionDef.State.IDLE,
                                                                 null,
                                                                 null,
                                                                 null));
             }

             for (int idleBoundSessionId : idleBoundSessions)
             {
                MultiSessionClientConnection connection =
                   msaManager.getConnection(connIdByBoundSessionId.get(idleBoundSessionId));
                if (connection == null)
                {
                   continue;
                }
                sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                                                 idleBoundSessionId,
                                                                 sessionCreateTime.get(idleBoundSessionId),
                                                                 true,
                                                                 MultiSessionAgentSessionDef.State.IDLE,
                                                                 connection.getModel().toString(),
                                                                 null,
                                                                 null));
             }

             for (int reservedSessionId : connIdByReservedSessionId.keySet())
             {
                MultiSessionClientConnection connection =
                   msaManager.getConnection(connIdByReservedSessionId.get(reservedSessionId));
                if (connection == null)
                {
                   continue;
                }
                sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                                                 reservedSessionId,
                                                                 sessionCreateTime.get(reservedSessionId),
                                                                 connIdByBoundSessionId.containsKey(reservedSessionId),
                                                                 MultiSessionAgentSessionDef.State.RESERVED,
                                                                 connection.getModel().toString(),
                                                                 null,
                                                                 null));
             }

             for (int activeSessionId : connIdByActiveSessionId.keySet())
             {
                MultiSessionClientConnection connection =
                   msaManager.getConnection(connIdByActiveSessionId.get(activeSessionId));
                if (connection == null)
                {
                   continue;
                }
                sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                                                 activeSessionId,
                                                                 sessionCreateTime.get(activeSessionId),
                                                                 connIdByBoundSessionId.containsKey(activeSessionId),
                                                                 MultiSessionAgentSessionDef.State.ACTIVE,
                                                                 connection.getModel().toString(),
                                                                 connection.getCurrentlyRunningProgram(),
                                                                 connection.getCurrentlyRunningProgramStartTime()));
             }
          }

          return sessionInfos.toArray(new MultiSessionAgentSessionDef[0]);
       }

       /**
        * Binds the client connection to the agent session if the session is not bound already. In Session-free
        * only persistent procedures are bound (to a child connection).
        *
        * @param    connection
        *           The client connection to get bound.
        * @param    sessionId
        *           The agent session id to get bound.
        * @param    isPersistentFunc
        *           True if a persistent function is initiating the bound.
        */
       void bind(MultiSessionClientConnection connection, Integer sessionId, boolean isPersistentFunc)
       {
          if ((connection.isSessionFree() && !isPersistentFunc) || connIdByBoundSessionId.containsKey(sessionId))
          {
             return;
          }
          synchronized (sessionsLock)
          {
             connIdByBoundSessionId.put(sessionId, connection.getId());
          }
          connection.setBoundSessionId(sessionId);
       }

       /**
        * Unbinds the client connection from the agent session if it's already bound. In Session-free only
        * persistent procedures are bound (to a child connection).
        *
        * @param    connection
        *           The client connection to get bound.
        * @param    sessionId
        *           The agent session id to get bound.
        * @param    isPersistentFunc
        *           True if a persistent function is initiating the bound.
        */
       void unbind(MultiSessionClientConnection connection, Integer sessionId, boolean isPersistentFunc)
       {
          if ((connection.isSessionFree() && !isPersistentFunc) || !connection.isBound())
          {
             return;
          }
          synchronized (sessionsLock)
          {
             String foundConnectionId = connIdByBoundSessionId.remove(sessionId);
             if (!foundConnectionId.equals(connection.getId()))
             {
                // indicates bug in the logic
                LOG.warning("Mismatched session removed from bound connection.");
             }
          }
          connection.setBoundSessionId(null);
          // TODO: delete all remote persistent procedures running in the session
       }

       /**
        * Returns <code>true</code> if the appserver is running.
        *
        * @return   See above.
        */
       boolean isRunning()
       {
          return state == State.RUNNING;
       }

       /**
        * Returns a Future for the reserved session id. Bound connections reserve the respective bound session,
        * and if it's already in use, a session listener is registered waiting it to be freed. The Future is
        * immediately returned and completed on reserving the freed session. For unbound connections the first
        * session in the idle sessions collection is reserved. If no sessions are idle, a new session is
        * created. If the max sessions number is reached, a session listener is registered waiting for any
        * unbound session to get freed.
        *
        * @param    connection
        *           The client connection.
        *
        * @return   Future for the session id.
        */
       Future<Integer> reserveSession(MultiSessionClientConnection connection)
       {
          if (state == State.TERMINATED || state == State.TERMINATING)
          {
             return NO_SESSION_FOUND_RESULT;
          }

          String connectionId = connection.getId();
          Integer boundSessionId = connection.getBoundSessionId();

          synchronized (sessionsLock)
          {
             if (boundSessionId != null)
             {
                if (!connIdByBoundSessionId.containsKey(boundSessionId))
                {
                   // the MSA has been manually or auto removed
                   LOG.warning("MSA session not found for bound connection.");
                   return NO_SESSION_FOUND_RESULT;
                }

                if (idleBoundSessions.contains(boundSessionId))
                {
                   reserveSession(connectionId, boundSessionId);
                   return CompletableFuture.completedFuture(boundSessionId);
                }

                if (!connIdByReservedSessionId
                   .containsKey(boundSessionId) && !connIdByActiveSessionId.containsKey(boundSessionId))
                {
                   // indicates bug in the logic
                   LOG.warning("MSA session not found for bound connection.");
                   return NO_SESSION_FOUND_RESULT;
                }

                return registerSessionListener(connectionId, boundSessionId);
             }

             if (idleSessions.size() > 0)
             {
                Integer idleSessionId = idleSessions.first();
                reserveSession(connectionId, idleSessionId);
                return CompletableFuture.completedFuture(idleSessionId);
             }

             if (connIdByActiveSessionId.size() + connIdByReservedSessionId.size() + idleSessions.size() +
                idleBoundSessions.size() >= definition.getMaxMsaSessionsPerAgent())
             {
                LOG.warning("The max number of MSA sessions is reached. Waiting for a session to get freed.");
                return registerSessionListener(connectionId, null);
             }
          }

          int newSessionId = createSession();
          try
          {
             MsaTask task = createSessionStartupTask(MultiSessionAgent.SINGLE_AGENT_ID,
                                                     newSessionId,
                                                     agentAdminConnection,
                                                     false,
                                                     false);
             submitTask(task).get();
             synchronized (sessionsLock)
             {
                // don't use freeSession() and reserveSession(),
                // because the session can be snatched by another request
                connIdByActiveSessionId.remove(newSessionId);
                connIdByReservedSessionId.put(newSessionId, connectionId);
             }
             return CompletableFuture.completedFuture(newSessionId);
          }
          catch (Exception e)
          {
             AppServerOperatingMode mode = connection.isSessionFree() ?
                AppServerOperatingMode.STATE_FREE :
                AppServerOperatingMode.STATELESS;
             String log = "Could not get and/or initialize '" + mode + "' session! Cannot process request.";
             outOfContextLogManager.writeMessage(log,
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_AUX,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             outOfContextLogManager.writeMessage("Error handling request! Status=-1003",
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_AUX,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             shutdownSession(MultiSessionAgent.SINGLE_AGENT_ID, newSessionId);
             return NO_SESSION_FOUND_RESULT;
          }
       }

       /**
        * Wraps a generic execution of a request received as a Runnable. A new MsaTask is created, submitted to
        * the executor and finally the session is freed. The MsaTask wraps starting the Runnable in a common
        * code that manages a top 'startup' scope, sets CURRENT-REQUEST-INFO, sets the return value or error as
        * the result.
        *
        * @param    connection
        *           The client connection.
        * @param    sessionId
        *           The session id.
        * @param    invokeRunnable
        *           The runnable to be started.
        *
        * @throws   Exception
        *           Any exception occurring during execution.
        */
       void runInTopScope(MultiSessionClientConnection connection, int sessionId, Runnable invokeRunnable)
       throws Exception
       {
          runInTopScope(null, null, connection, sessionId, null, invokeRunnable, null, null);
       }

       /**
        * Wraps a generic execution of a request received as a Runnable. A new MsaTask is created, submitted to
        * the executor and finally the session is freed. The MsaTask wraps starting the Runnable in a common
        * code that manages a top 'startup' scope, sets CURRENT-REQUEST-INFO, sets the return value or error as
        * the result.
        *
        * @param    result
        *           The result of the appserver call.
        * @param    requestInfo
        *           The request info data.
        * @param    connection
        *           The client connection.
        * @param    sessionId
        *           The session id.
        * @param    runningProgram
        *           The name of the program to be executed.
        * @param    invokeRunnable
        *           The runnable to be started.
        * @param    isTaskRunning
        *           Thread-safe supplier indicating if the wrapper task is running.
        * @param    isTaskCancelled
        *           Thread-safe supplier indicating if the wrapper task is cancelled.
        *
        * @throws   Exception
        *           Any exception occurring during execution.
        */
       void runInTopScope(AppServerInvocationResult result,
                          Object[] requestInfo,
                          MultiSessionClientConnection connection,
                          int sessionId,
                          String runningProgram,
                          Runnable invokeRunnable,
                          AtomicBoolean isTaskRunning,
                          AtomicBoolean isTaskCancelled)
       throws Exception
       {
          MsaTask task = new MsaTask(definition.getAppserverName(),
                                     MultiSessionAgent.SINGLE_AGENT_ID,
                                     sessionId,
                                     connection,
                                     result,
                                     runningProgram,
                                     isBound(connection.getId()),
                                     isTaskRunning,
                                     isTaskCancelled,
                                     false)
          {
             @Override
             public void coreRun()
             {
                // add the topmost scope to the TransactionManager
                TransactionManager.pushScope("startup",
                                             TransactionManager.NO_TRANSACTION,
                                             true,
                                             true,
                                             false,
                                             false);

                if (requestInfo != null && requestInfo.length > 0)
                {
                   SessionUtils.currentRequestInfo().ref().fromArray(requestInfo);
                }

                try
                {
                   invokeRunnable.run();
                }
                catch (Throwable t)
                {
                   if (!UnstoppableExitException.isCausing(t))
                   {
                      LOG.log(Level.WARNING,
                              t,
                              "runInTopScope has failed for appserver %s:",
                              definition.getAppserverName());
                      if (result != null)
                      {
                         result.setError(t);
                      }
                   }
                }
                finally
                {
                   boolean isContextDeleted = TransactionManager.getScopeLabel() == null;
                   if (!isContextDeleted)
                   {
                      // remove the topmost scope from the TransactionManager
                      TransactionManager.popScope();
                   }
                }

                if (ControlFlowOps.isReturnValueSetInRootScope() && result != null)
                {
                   result.setResult(ControlFlowOps.getReturnValue());
                }
             }
          };

          try
          {
             submitTask(task).get();
          }
          finally
          {
             freeSession(sessionId, true);
          }
       }

       /**
        * A simple wrapper for a generic execution of a request in the PASOE context. A new MsaTask is created,
        * submitted to the executor and finally the session is freed.
        *
        * @param    connection
        *           The client connection.
        * @param    sessionId
        *           The session id.
        * @param    result
        *           The result of the appserver call.
        * @param    runnable
        *           The runnable to be started.
        * @param    isTaskRunning
        *           Thread-safe supplier indicating if the MSA task is running.
        * @param    isTaskCancelled
        *           Thread-safe supplier indicating if the wrapper task is cancelled.
        * @param    taskReceiver
        *           Function called to supply the newly created MsaTask to the caller.
        *
        * @throws   Exception
        *           Any exception occurring during execution.
        */
       void runInSessionContext(MultiSessionClientConnection connection,
                                int sessionId,
                                AppServerInvocationResult result,
                                Runnable runnable,
                                AtomicBoolean isTaskRunning,
                                AtomicBoolean isTaskCancelled,
                                Function<MsaTask, Void> taskReceiver)
       throws Exception
       {
          MsaTask task = new MsaTask(definition.getAppserverName(),
                                     MultiSessionAgent.SINGLE_AGENT_ID,
                                     sessionId,
                                     connection,
                                     result,
                                     null,
                                     isBound(connection.getId()),
                                     isTaskRunning,
                                     isTaskCancelled,
                                     false)
          {
             @Override
             void coreRun()
             {
                runnable.run();
             }
          };

          if (taskReceiver != null)
          {
             taskReceiver.apply(task);
          }

          try
          {
             submitTask(task).get();
          }
          finally
          {
             freeSession(sessionId, true);
          }
       }

       /**
        * Terminate the appserver by running the termination tasks for all agents.
        *
        * @param    exitCode
        *           The exit code for the appserver.
        */
       void terminateAppserver(MultiSessionAgent.ExitCode exitCode)
       {
          terminateAgent(MultiSessionAgent.SINGLE_AGENT_ID, exitCode);
       }

       /**
        * Terminate the agent by running the termination task.
        *
        * @param    exitCode
        *           The exit code for the appserver.
        */
       void terminateAgent(int agentId, MultiSessionAgent.ExitCode exitCode)
       {
          // If multiple agents are implemented sweeping an idle agent should set the timeout to
          // definition.getDefaultAgentWaitToFinish() + definition.getDefaultAgentWaitAfterStop()
          new AgentTerminationTask(agentId, definition.getCompleteActiveReqTimeout(), exitCode, false).run();
       }

       /**
        * Terminate the agent session. Cleans up all references to the session. If the session is currently in
        * use, registers a session listener. Runs the session shutdown procedure. If requested, finalizes the FWD
        * security context. If it's the last session on the agent and the flag is raised, it also terminates
        * the agent.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionId
        *           The session id.
        * @param    finalizeCurrentContext
        *           Flag to indicate if the current context needs to be finalized after the session shut down.
        * @param    finalizeAgent
        *           Flag to indicate if the current agent needs to be terminated after the session shut down
        *           (in effect only when the last session in the agent is shut down).
        *
        * @return   Future that gets completed when the termination of the session is done.
        */
       Future<?> terminateSession(int agentId,
                                  int sessionId,
                                  boolean finalizeCurrentContext,
                                  boolean finalizeAgent)
       {
          Future<Integer> freeSessionFuture = null;

          synchronized (sessionsLock)
          {
             boundSessionListeners.removeIf(l -> l.requestedSessionId == sessionId);

             if (connIdByActiveSessionId.containsKey(sessionId))
             {
                String connectionId = connIdByActiveSessionId.get(sessionId);
                if (connectionId != null)
                {
                   MultiSessionClientConnection connection = MultiSessionAppserverManager.getInstance()
                                                                                         .getConnection(connectionId);
                   freeSessionFuture =
                      registerSessionListener(connection == null ? null : connection.getId(), sessionId);

                   if (connection != null)
                   {
                      connection.stopCurrentTask();
                   }
                }
             }
             else
             {
                idleSessions.remove(sessionId);
                idleBoundSessions.remove(sessionId);
                connIdByReservedSessionId.remove(sessionId);
             }

             connIdByBoundSessionId.remove(sessionId);
          }

          if (freeSessionFuture != null)
          {
             try
             {
                // TODO; timeout?
                freeSessionFuture.get();
             }
             catch (Exception ignored)
             {
             }
          }

          try
          {
             return shutdownSession(agentId,
                                    sessionId,
                                    agentAdminConnection,
                                    finalizeCurrentContext,
                                    finalizeAgent,
                                    false);
          }
          finally
          {
             freeSession(sessionId, false, true);
          }
       }

       /**
        * Delete the remote procedures active on the session and connection. This must be called at the same
        * time when the local proxy procedure is deleted.
        *
        * @param    connection
        *           The client connection.
        * @param    shutdownBoundSession
        *           Flag to indicate the bound MSA session should be shut down.
        */
       void deleteConnection(MultiSessionClientConnection connection, boolean shutdownBoundSession)
       {
          boolean hasPersistentProc = connection.hasPersistentProc();
          for (AppserverRemoteProcedure remote : connection.getPersistentProcs())
          {
             Object referent = remote.getExtProg().get();
             if (referent instanceof _BaseObject_)
             {
                connection.deletePersistentObject(remote.getExtProgResourceId());
             }
             else
             {
                connection.deletePersistentProc(remote.getExtProgResourceId());
             }
          }

          if (shutdownBoundSession && connection.isBound())
          {
             shutdownSession(MultiSessionAgent.SINGLE_AGENT_ID, connection.getBoundSessionId());
          }

          String connectionId = connection.getId();
          clearConnection(connectionId, shutdownBoundSession);

          if (!connection.isSessionFree() || hasPersistentProc)
          {
             log("Application Server disconnected with connection id: " + connectionId + ". (8359)",
                 LoggingLevel.BASIC,
                 LoggingEntryType.DB_Connects,
                 LoggingExecEnv.AS_ADMIN);
          }
       }

       /**
        * Iterates over all bound connections searching to delete the specified connection by its id. The bound
        * session is removed from the collection of idle bound sessions and moved to the idle unbound sessions,
        * if the the session is not about to be shut down.
        *
        * @param    connectionId
        *           The connection id.
        * @param    shutdownBoundSession
        *           Flag to indicate the bound MSA session is to be shut down.
        */
       void clearConnection(String connectionId, boolean shutdownBoundSession)
       {
          synchronized (sessionsLock)
          {
             for (Iterator<Map.Entry<Integer, String>> it = connIdByBoundSessionId.entrySet().iterator();
                  it.hasNext();)
             {
                Map.Entry<Integer, String> entry = it.next();
                if (entry.getValue().equals(connectionId))
                {
                   int boundSessionId = entry.getKey();
                   idleBoundSessions.remove(boundSessionId);
                   if (!shutdownBoundSession)
                   {
                      idleSessions.add(boundSessionId);
                   }
                   it.remove();
                }
             }
          }
       }

       /**
        * Iterates over all idle sessions and removes them one by one, running each shutdown task.
        *
        * @param    agentId
        *           The agent id.
        */
       void removeIdleSessions(int agentId)
       {
          synchronized (sessionsLock)
          {
             for (Iterator<Integer> it = idleSessions.iterator(); it.hasNext();)
             {
                shutdownNextSession(agentId, it, true, false, false);
             }
          }
       }

       /**
        * Writes a legacy log record to the appserver log file using a logger out of the appserver context.
        *
        * @param    msg
        *           The log message.
        * @param    level
        *           The LOG-MANAGER log level.
        * @param    entryType
        *           The type of the entry.
        * @param    execEnv
        *           The LOG-MANAGER execution env.
        */
       void log(String msg,
                LoggingLevel level,
                LoggingEntryType entryType,
                LoggingExecEnv execEnv)
       {
          outOfContextLogManager.writeMessage(msg, entryType, execEnv, LoggingSubsys.AS, level);
       }

       /**
        * Finds the LOG-MANAGER configurations for this appserver, instantiates a new logger out of the appserver
        * context and initializes the logger.
        */
       private void prepareLogging()
       {
          // replicates AppServerManager.startAppServer
          LegacyLogManagerConfigs legacyLogManagerConfigs =
             LegacyLogOps.appserverNameLogConfigPairs.get(definition.getAppserverName());

          outOfContextLogManager = LegacyLogOps.newStaticLogMgr();
          outOfContextLogManager.initialize(legacyLogManagerConfigs);
       }

       /**
        * Runs all necessary actions to setup a new appserver agent.
        *
        * @throws   Exception
        *           Any exception thrown during agent initialization.
        */
       private void initializeAgent()
       throws Exception
       {
          int agentId = MultiSessionAgent.SINGLE_AGENT_ID;

          //TODO: what about definition.getAgentStartupParam() ?
          startupParameters = StandardServer.initializeStartupParameters(new StartupParameters());

          taskQueue = new LinkedBlockingQueue<>();
          workerPoolExecutor = new ThreadPoolExecutor(definition.getMaxConnectionsPerAgent(),
                                                      definition.getMaxConnectionsPerAgent(),
                                                      definition.getIdleConnectionTimeout(),
                                                      TimeUnit.MILLISECONDS,
                                                      taskQueue,
                                                      new MsaThreadFactory(outOfContextLogManager));

          workerPoolExecutor.allowCoreThreadTimeOut(true);
          workerPoolExecutor.prestartAllCoreThreads();

          int firstSessionId = createSession();
          try
          {
             startupSession(agentId, firstSessionId, agentAdminConnection, true, false);
          }
          catch (Throwable t)
          {
             outOfContextLogManager.writeMessage("Failed to get or initialize first user session.",
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_AUX,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             outOfContextLogManager.writeMessage("agentStartupProc Failed -- Shutting down...",
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_LISTENER,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             LOG.warning("Initializing the MSA agent with id " + agentId + " failed for appserver " +
                            definition.getAppserverName(), t);
             terminateAgent(agentId, MultiSessionAgent.ExitCode.UNEXPECTED_1003);

             // need to cast to Exception to comply with the FutureTask interface
             Throwable cause = t.getCause();
             if (cause != null)
             {
                throw (Exception) cause;
             }
             else
             {
                throw (Exception) t;
             }
          }

          for (int i = MIN_ABL_SESSIONS; i < definition.getNumInitialSessions(); i++)
          {
             int sessionId = createSession();
             try
             {
                startupSession(agentId, sessionId, agentAdminConnection, false, false);
             }
             catch (Throwable t)
             {
                // TODO: test if secondary sessions failure stops server init
                // need to cast to Exception to comply with the FutureTask interface
                Throwable cause = t.getCause();
                if (cause != null)
                {
                   throw (Exception) cause;
                }
                else
                {
                   throw (Exception) t;
                }
             }
          }
       }

       /**
        * Runs all necessary actions to setup the appserver session.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionId
        *           The id of the FWD context to be used for the session.
        * @param    connection
        *           The client connection.
        * @param    runAgentStartupProc
        *           Flag to indicate it's the first session in the agent and the agent startup procedure needs
        *           to run.
        * @param    sessionReset
        *           Flag to indicate the setup is part of a session reset.
        *
        * @throws   Exception
        *           Any exception thrown during agent initialization.
        */
       private void startupSession(int agentId,
                                   int sessionId,
                                   MultiSessionClientConnection connection,
                                   boolean runAgentStartupProc,
                                   boolean sessionReset)
       throws Exception
       {
          MsaTask task = createSessionStartupTask(MultiSessionAgent.SINGLE_AGENT_ID,
                                                  sessionId,
                                                  connection,
                                                  runAgentStartupProc,
                                                  sessionReset);

          try
          {
             submitTask(task).get();
          }
          catch (Throwable t)
          {
             shutdownSession(agentId, sessionId);
             throw t;
          }

          freeSession(sessionId, true);
       }

       /**
        * Creates an MsaTask running all necessary actions to setup the session.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionId
        *           The id of the FWD context to be used for the session.
        * @param    connection
        *           The client connection.
        * @param    runAgentStartupProc
        *           Flag to indicate it's the first session in the agent and the agent startup procedure needs
        *           to run.
        * @param    sessionReset
        *           Flag to indicate the setup is part of a session reset.
        *
        * @return   The MsaTask to run the session setup.
        */
       private MsaTask createSessionStartupTask(int agentId,
                                                int sessionId,
                                                MultiSessionClientConnection connection,
                                                boolean runAgentStartupProc,
                                                boolean sessionReset)
       {
          return new MsaTask(definition.getAppserverName(),
                             agentId,
                             sessionId,
                             connection,
                             null,
                             null,
                             false,
                             null,
                             null,
                             MultiSessionClientConnection.AGENT_SYS_CONN_ID.equals(connection.getId()))
          {
             @Override
             public void coreRun()
             {
                if (!sessionReset)
                {
                   String msg = "Starting MSAS Session for " + definition.getAppserverName() + ".";
                   outOfContextLogManager.writeMessage(msg,
                                                       LoggingEntryType.ASPlumbing,
                                                       null,
                                                       LoggingSubsys.AS,
                                                       LoggingLevel.BASIC);
                }

                // replicates AgentPool.start
                Utils.setProjectToken(definition.getProjectToken());

                // resolve clientConfig/cfgOverrides
                ConfigItem.resolveCfgOverrides(Utils.DirScope.BOTH, definition.getAccountIds());

                // replicates ClientCore.initialize
                ClientParameters clientParameters = definition.getClientParameters();
                clientParameters.socketTrustStoreFilename = ConfigItem.SOCKET_TRUSTSTORE_FILE
                   .read("trusted-cert.store");
                clientParameters.socketTrustStorePassword = ConfigItem.SOCKET_TRUSTSTORE_PASS.read("");

                StandardServer.setClientParams(clientParameters);

                // replicates StandardServer.initializeStartupParameters
                SessionUtils.setStartupParameters(startupParameters);

                // replicates StandardServer.setupSession
                SessionUtils.setDeviceId(clientParameters.osDeviceId);
                SessionUtils.setClientTimezone(clientParameters.clientTimezone);

                // replicates Agent.prepare
                CoreAppserver.prepareSession(true,
                                             definition.getAppserverName(),
                                             definition.getPropath(),
                                             SESSION_GLOBAL_SCOPE);

                String agentStartupProc = definition.getAgentStartupProc();
                String sessionStartupProc = definition.getSessionStartupProc();

                if (!sessionReset)
                {
                   outOfContextLogManager.writeMessage("MSAS Session Startup. (5473)",
                                                       LoggingEntryType.ASPlumbing,
                                                       null,
                                                       LoggingSubsys.AS,
                                                       LoggingLevel.BASIC);
                }

                if ((!runAgentStartupProc || agentStartupProc == null) && sessionStartupProc == null)
                {
                   // no startup procs, initialization completed
                   return;
                }

                // add the topmost scope to the TransactionManager
                TransactionManager.pushScope("startup",
                                             TransactionManager.NO_TRANSACTION,
                                             true,
                                             true,
                                             false,
                                             false);

                if (runAgentStartupProc && agentStartupProc != null)
                {
                   runStartupProc(true, agentStartupProc, definition.getAgentStartupProcParam());
                }

                if (sessionStartupProc != null)
                {
                   runStartupProc(false, sessionStartupProc, definition.getSessionStartupProcParam());
                }

                TransactionManager.popScope(); // remove the "startup" scope
             }
          };
       }

       /**
        * Runs the agent or session startup procedure.
        *
        * @param    isAgentStartupProc
        *           <code>true</code> if it's the agent startup procedure, <code>false</code> if it's the session
        *           startup procedure.
        * @param    startupProc
        *           The startup procedure.
        * @param    startupProcParam
        *           The startup procedure parameters.
        */
       private void runStartupProc(boolean isAgentStartupProc, String startupProc, String startupProcParam)
       {
          try
          {
             ControlFlowOps.invokePersistentSetWithMode(startupProc,
                                                        new handle(),
                                                        "I",
                                                        new character(startupProcParam));
          }
          catch (Throwable t)
          {
             if (!UnstoppableExitException.isCausing(t))
             {
                LegacyLogManager logManager = LegacyLogOps.logMgr();
                logStartupShutdownCondition(startupProc, t, logManager, true);

                int error = -6;

                if (t instanceof ErrorConditionException &&
                   ((ErrorConditionException) t).getProgressErrorCode() == 8024)
                {
                   error = -1;
                }

                String startupProcType = isAgentStartupProc ? "agent" : "session";
                logManager.writeMessage("Error (" + error + ") running " + startupProcType + " startup procedure.",
                                        LoggingEntryType.ASPlumbing,
                                        null,
                                        LoggingSubsys.MSAS,
                                        LoggingLevel.ERROR);

                logManager.writeMessage("Unable to initialize session!",
                                        LoggingEntryType.ASPlumbing,
                                        null,
                                        LoggingSubsys.MSAS,
                                        LoggingLevel.ERROR);

                if (LOG.isLoggable(Level.WARNING))
                {
                   LOG.log(Level.WARNING,
                           t,
                           "Running " + startupProcType + " startup procedure %s has failed for appserver %s:",
                           startupProc,
                           definition.getAppserverName());
                }

                TransactionManager.popScope(); // remove the "startup" scope

                throw t;
             }
          }
       }

       /**
        * Creates and saves a session listener, then returns a Future that gives the id of the session
        * received by the listener, when the session got free. Bound connections request a listener for a
        * specific session, while unbound connections wait in line to receive a notification for the next free
        * unbound session.
        *
        * @param    connectionId
        *           The client connection id.
        * @param    sessionId
        *           The agent session id.
        *
        * @return   Future returning the session id.
        */
       private Future<Integer> registerSessionListener(String connectionId, Integer sessionId)
       {
          if (state == State.TERMINATING)
          {
             return NO_SESSION_FOUND_RESULT;
          }

          SessionListenerFuture sessionListenerFuture = new SessionListenerFuture(connectionId, sessionId);

          synchronized (sessionsLock)
          {
             if (sessionId == null)
             {
                unboundSessionListeners.add(sessionListenerFuture);
             }
             else
             {
                boundSessionListeners.add(sessionListenerFuture);
             }
          }
          return sessionListenerFuture;
       }

       /**
        * Creates a new FWD context for a multi-session agent session and returns the id.
        *
        * @return   The id of the newly created session (MSA security context). {@link #INVALID_SESSION_ID} if
        *           a RestrictedUseException was thrown during creation.
        */
       private int createSession()
       {
          try
          {
             int contextId = msaSecurityManager.createContext(definition.getAppserverName(),
                                                              definition.getAccountIds()[0]);
             sessionCreateTime.put(contextId, new Date());
             return contextId;
          }
          catch (RestrictedUseException e)
          {
             LOG.severe("RestrictedUseException in createSession.");
          }
          return INVALID_SESSION_ID;
       }

       /**
        * Sets the FWD context to the thread.
        *
        * @return   <code>true</code> if the context has been set successfully, <code>false</code> otherwise.
        */
       private boolean switchSession(String appserverName, int sessionId)
       {
          try
          {
             return msaSecurityManager.setContext(appserverName, sessionId);
          }
          catch (RestrictedUseException e)
          {
             LOG.severe("RestrictedUseException in switchSession.");
          }
          return false;
       }

       /**
        * Moves an agent session from idle to reserved state.
        *
        * @param    connectionId
        *           The client connection id.
        * @param    sessionId
        *           The agent session id.
        */
       private void reserveSession(String connectionId, int sessionId)
       {
          synchronized (sessionsLock)
          {
             if (isBound(sessionId))
             {
                idleBoundSessions.remove(sessionId);
             }
             else
             {
                idleSessions.remove(sessionId);
             }
             connIdByReservedSessionId.put(sessionId, connectionId);
          }
       }

       /**
        * Moves an agent session from reserved to active state.
        *
        * @param    connectionId
        *           The client connection id.
        * @param    sessionId
        *           The agent session id.
        */
       private void useSession(String connectionId, int sessionId)
       {
          synchronized (sessionsLock)
          {
             connIdByActiveSessionId.put(sessionId, connectionId);
             connIdByReservedSessionId.remove(sessionId);
          }
       }

       /**
        * See {@link #freeSession(int, boolean, boolean)}.
        *
        * @param    sessionId
        *           The agent session id.
        * @param    callSessionListeners
        *           Flag to indicate the session listeners need to be called. <code>false</code> during session
        *           termination.
        */
       private void freeSession(int sessionId, boolean callSessionListeners)
       {
          freeSession(sessionId, callSessionListeners, false);
       }

       /**
        * Sets the agent session state to free. Removes the session from the active and reserved session
        * collections.
        * <br>
        * If in the meantime the appserver has been terminated, the session is in the process of
        * shutdown, or the FWD context has been deleted, then the operation stops. If the appserver is in the
        * process of terminating {@link #terminationSessionListener} is called.
        * <br>
        * If the max MSA session age is configured and the session is not bound, then a check if performed to
        * decide if the session needs to be shutdown instead of added back to the collection of free sessions.
        * <br>
        * Bound and unbound sessions are handled in a similar way, but they are recorded in their respective
        * idle session collection and call their respective session listeners.
        *
        * @param    sessionId
        *           The agent session id.
        * @param    callSessionListeners
        *           Flag to indicate the session listeners need to be called. <code>false</code> during session
        *           termination.
        * @param    isSessionShutdown
        *           Flag to indicate the session is to be shut down.
        */
       private void freeSession(int sessionId, boolean callSessionListeners, boolean isSessionShutdown)
       {
          boolean isBound;
          synchronized (sessionsLock)
          {
             connIdByActiveSessionId.remove(sessionId);
             connIdByReservedSessionId.remove(sessionId);

             if (state == State.TERMINATED)
             {
                return;
             }

             boolean isDeleted = !msaSecurityManager.hasContext(definition.getAppserverName(), sessionId);
             if (isDeleted)
             {
                return;
             }

             if (state == State.TERMINATING)
             {
                if (terminationSessionListener != null)
                {
                   terminationSessionListener.onSessionFreed(sessionId);
                }
                return;
             }

             if (isSessionShutdown)
             {
                return;
             }

             isBound = isBound(sessionId);
          }

          if (!isBound && definition.getMaxMsaSessionAge() > 0)
          {
             Date sessionCreateTime = this.sessionCreateTime.get(sessionId);
             if (sessionCreateTime.getTime() + definition.getMaxMsaSessionAge() < new Date().getTime())
             {
                if (LOG.isLoggable(Level.FINE))
                {
                   LOG.fine("Old MSA session with id " + sessionId + " is to be shutdown.");
                }
                shutdownSession(MultiSessionAgent.SINGLE_AGENT_ID,
                                sessionId,
                                agentAdminConnection,
                                true,
                                false,
                                false);
                return;
             }
          }

          synchronized (sessionsLock)
          {
             if (!isBound)
             {
                idleSessions.add(sessionId);
                if (callSessionListeners)
                {
                   Iterator<SessionListenerFuture> iterator = unboundSessionListeners.iterator();
                   while (iterator.hasNext())
                   {
                      SessionListener sessionListener = iterator.next();
                      if (sessionListener.isCancelled())
                      {
                         iterator.remove();
                         continue;
                      }
                      iterator.remove();
                      sessionListener.onSessionFreed(sessionId);
                   }
                }
                return;
             }

             idleBoundSessions.add(sessionId);

             if (callSessionListeners)
             {
                Iterator<SessionListenerFuture> iterator = boundSessionListeners.iterator();
                while (iterator.hasNext())
                {
                   SessionListener sessionListener = iterator.next();
                   Integer boundSession = sessionListener.getRequestedSessionId();
                   if (sessionListener.isCancelled())
                   {
                      iterator.remove();
                      continue;
                   }
                   if (boundSession == sessionId)
                   {
                      iterator.remove();
                      sessionListener.onSessionFreed(sessionId);
                      break;
                   }
                }
             }
          }
       }

       /**
        * Returns <code>true</code> if the connection is bound, <code>false</code> otherwise.
        *
        * @param    connectionId
        *           The client connection id.
        *
        * @return   See above.
        */
       private boolean isBound(String connectionId)
       {
          return connIdByBoundSessionId.containsValue(connectionId);
       }

       /**
        * Returns <code>true</code> if the connection is bound, <code>false</code> otherwise.
        *
        * @param    sessionId
        *           The agent session id.
        *
        * @return   See above.
        */
       private boolean isBound(int sessionId)
       {
          return connIdByBoundSessionId.containsKey(sessionId);
       }

       /**
        * Submits a {@link MsaTask} for execution. Tasks are not accepted during agent termination.
        *
        * @param    task
        *           The MSA task to be executed.
        *
        * @return   Future completed when the task execution is completed.
        *
        * @throws   Exception
        *           An exception thrown when the task can't be accepted.
        */
       private Future submitTask(MsaTask task)
       throws Exception
       {
          return submitTask(task, false);
       }

       /**
        * Submits a {@link MsaTask} for execution.
        *
        * @param    task
        *           The MSA task to be executed.
        * @param    acceptDuringTermination
        *           Flag to indicate the task should be accepted during agent termination.
        *
        * @return   Future completed when the task execution is completed.
        *
        * @throws   Exception
        *           An exception thrown when the task can't be accepted.
        */
       private Future submitTask(MsaTask task, boolean acceptDuringTermination)
       throws Exception
       {
          if (!acceptDuringTermination && (state == State.TERMINATING || state == State.TERMINATED))
          {
             // TODO: replace with the OE error
             throw new Exception("Can't accept new tasks. The appserver " + definition.getAppserverName()
                                    + " is not running.");
          }
          if (Thread.currentThread().getName().startsWith(WORKER_THREAD_NAME_PREFIX))
          {
             // sync exec on the same thread
             task.run();
             return CompletableFuture.completedFuture(null);
          }
          return workerPoolExecutor.submit(task);
       }

       /**
        * Shuts down the next session provided with the iterator.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionIdIterator
        *           Iterator for session ids. Used to determine the next session to be shut down.
        * @param    finalizeCurrentContext
        *           Flag to indicate if the current context needs to be finalized after the session shut down.
        * @param    finalizeAgent
        *           Flag to indicate if the current agent needs to be terminated after the session shut down
        *           (in effect only when the last session in the agent is shut down).
        * @param    acceptDuringTermination
        *           Flag to indicate the task should be accepted during agent termination.
        */
       private void shutdownNextSession(int agentId,
                                        Iterator<Integer> sessionIdIterator,
                                        boolean finalizeCurrentContext,
                                        boolean finalizeAgent,
                                        boolean acceptDuringTermination)
       {
          Integer sessionId = sessionIdIterator.next();
          sessionIdIterator.remove();
          shutdownSession(agentId,
                          sessionId,
                          agentAdminConnection,
                          finalizeCurrentContext,
                          finalizeAgent,
                          acceptDuringTermination);
       }

       /**
        * Clears all references to the session and runs the session shutdown task.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionId
        *           The session id to be shut down.
        */
       private void shutdownSession(int agentId, int sessionId)
       {
          synchronized (sessionsLock)
          {
             // should not be added to the idle sessions, so don't use freeSession()
             idleSessions.remove(sessionId);
             idleBoundSessions.remove(sessionId);
             connIdByBoundSessionId.remove(sessionId);
             connIdByReservedSessionId.remove(sessionId);
             connIdByActiveSessionId.remove(sessionId);
          }

          shutdownSession(agentId,
                          sessionId,
                          agentAdminConnection,
                          true,
                          true,
                          false);
       }

       /**
        * Runs the session shutdown task.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionId
        *           The session id to be shut down.
        * @param    connection
        *           The connection (client or system).
        * @param    finalizeCurrentContext
        *           Flag to indicate if the current context needs to be finalized after the session shut down.
        * @param    finalizeAgent
        *           Flag to indicate if the current agent needs to be terminated after the session shut down
        *           (in effect only when the last session in the agent is shut down).
        * @param    acceptDuringTermination
        *           Flag to indicate the task should be accepted during agent termination.
        *
        * @return   Future completed when the session shutdown task is done.
        */
       private Future<?> shutdownSession(int agentId,
                                         int sessionId,
                                         MultiSessionClientConnection connection,
                                         boolean finalizeCurrentContext,
                                         boolean finalizeAgent,
                                         boolean acceptDuringTermination)
       {
          // before the shutdown call the session id has been removed from idle, reserved and active lists
          boolean executeAgentShutdownProc =
             finalizeAgent && !hasAnySession() && definition.getAgentShutdownProc() != null;

          if (!MultiSessionAppserverSecurityManager.getInstance()
                                                   .hasContext(connection.getAppserverName(), sessionId))
          {
             LOG.log(Level.FINE, "Shutting down the session failed:");
             return CompletableFuture.completedFuture(false);
          }

          MsaTask task = new MsaTask(definition.getAppserverName(),
                                     agentId,
                                     sessionId,
                                     connection,
                                     null,
                                     null,
                                     false,
                                     null,
                                     null,
                                     MultiSessionClientConnection.AGENT_SYS_CONN_ID.equals(connection.getId()))
          {
             @Override
             void coreRun()
             {
                if (definition.getSessionShutdownProc() != null || executeAgentShutdownProc)
                {
                   boolean isNestedCall = "startup".equals(TransactionManager.getScopeLabel());
                   if (!isNestedCall)
                   {
                      // add the topmost scope to the TransactionManager
                      TransactionManager.pushScope("startup",
                                                   TransactionManager.NO_TRANSACTION,
                                                   true,
                                                   true,
                                                   false,
                                                   false);
                   }

                   if (definition.getSessionShutdownProc() != null)
                   {
                      try
                      {
                         String sessionShutdownProc = definition.getSessionShutdownProc();

                         log("shutdown Procedure '" + sessionShutdownProc + "' START (14244)",
                             LoggingLevel.VERBOSE,
                             LoggingEntryType.ASPlumbing,
                             null);

                         ControlFlowOps.invokeExternalProcedure(new character(sessionShutdownProc),
                                                                false,
                                                                null,
                                                                false,
                                                                null,
                                                                null);

                         log("shutdown Procedure END SUCCESS. (14265)",
                             LoggingLevel.VERBOSE,
                             LoggingEntryType.ASPlumbing,
                             null);
                      }
                      catch (Exception e)
                      {
                         logShutdownProcError(definition.getSessionShutdownProc(), e);

                         if (!executeAgentShutdownProc)
                         {
                            if (!isNestedCall)
                            {
                               // remove the topmost scope from the TransactionManager
                               TransactionManager.popScope();
                            }

                            if (finalizeCurrentContext)
                            {
                               finalizeCurrentContext(sessionId);
                            }

                            //throw e;
                            return; // task is running async without waiting for result, so just stop execution
                         }
                      }
                   }

                   if (executeAgentShutdownProc)
                   {
                      try
                      {
                         ControlFlowOps.invokeExternalProcedure(new character(definition.getAgentShutdownProc()),
                                                                false,
                                                                null,
                                                                false,
                                                                null,
                                                                null);
                      }
                      catch (Exception e)
                      {
                         if (!isNestedCall)
                         {
                            // remove the topmost scope from the TransactionManager
                            TransactionManager.popScope();
                         }

                         logShutdownProcError(definition.getAgentShutdownProc(), e);

                         if (finalizeCurrentContext)
                         {
                            finalizeCurrentContext(sessionId);
                         }

                         //throw e;
                         return; // task is running async without waiting for result, so just stop execution
                      }
                   }

                   if (!isNestedCall)
                   {
                      // remove the topmost scope from the TransactionManager
                      TransactionManager.popScope();
                   }
                }

                if (finalizeCurrentContext)
                {
                   finalizeCurrentContext(sessionId);
                }
             }

             private void logShutdownProcError(String procName, Exception e)
             {
                if (!UnstoppableExitException.isCausing(e))
                {
                   LegacyLogManager logManager = LegacyLogOps.logMgr();
                   logStartupShutdownCondition(procName, e, logManager, false);
                }
             }
          };

          try
          {
             return submitTask(task, acceptDuringTermination);
          }
          catch (Exception e)
          {
             LOG.log(Level.FINE, "Shutting down the session failed:", e);
          }

          return null;
       }

       /**
        * Creates a legacy log for the the exception thrown by running the startup / shutdown agent / session
        * procedure (all four variants).
        *
        * @param    procName
        *           The procedure name.
        * @param    throwable
        *           The throwable to be handled.
        * @param    logManager
        *           The instance of the LOG-MANAGER.
        * @param    isStartupProc
        *           <code>true</code> if it's a startup procedure (agent or session).
        */
       private void logStartupShutdownCondition(String procName,
                                                Throwable throwable,
                                                LegacyLogManager logManager,
                                                boolean isStartupProc)
       {
          BlockManager.Condition conditionType;
          int errorNumber;
          if (throwable instanceof StopConditionException)
          {
             conditionType = BlockManager.Condition.STOP;
             errorNumber = 8026;
          }
          else if (throwable instanceof QuitConditionException)
          {
             conditionType = BlockManager.Condition.QUIT;
             errorNumber = 8027;
          }
          else
          {
             conditionType = BlockManager.Condition.ERROR;
             errorNumber = 8025;
          }

          String msg = procName +
             (isStartupProc ? " startup" : " shutdown") +
             " procedure ended with an " + conditionType.name() +
             " condition. (" + errorNumber + ")";
          logManager.writeMessage(msg,
                                  LoggingEntryType.ASPlumbing,
                                  null,
                                  LoggingSubsys.EMPTY,
                                  LoggingLevel.ERROR);
       }

       /**
        * Runnable handling the agent termination.
        */
       private class AgentTerminationTask
       implements Runnable
       {
          /** The name of the appserver the agent belongs to. */
          private final String appserverName;

          /** The agent id. */
          private final int agentId;

          /** Timeout in milliseconds before the task is cancelled. */
          private final int timeoutMs;

          /** The exit code. */
          private final MultiSessionAgent.ExitCode exitCode;

          /** Flag indicating if the JVM is shutting down.. */
          private final boolean isJvmShuttingDown;

          /**
           * Package-private constructor.
           *
           * @param    agentId
           *           The agent id.
           * @param    timeoutMs
           *           Timeout in milliseconds before the task is cancelled.
           * @param    exitCode
           *           The exit code.
           * @param    isJvmShuttingDown
           *           Flag indicating if the JVM is shutting down.
           */
          AgentTerminationTask(int agentId,
                               int timeoutMs,
                               MultiSessionAgent.ExitCode exitCode,
                               boolean isJvmShuttingDown)
          {
             this.appserverName = definition.getAppserverName();
             this.agentId = agentId;
             this.timeoutMs = timeoutMs;
             this.exitCode = exitCode;
             this.isJvmShuttingDown = isJvmShuttingDown;
          }

          /**
           * Checks if the argument object represents the same task, identified by the all its fields.
           *
           * @param    o
           *           The object to compare to.
           *
           * @return   <code>true</code> if it's the same instance or all the fields are the same.
           *           <code>false</code> otherwise.
           */
          @Override
          public boolean equals(Object o)
          {
             if (this == o)
             {
                return true;
             }
             if (o == null || getClass() != o.getClass())
             {
                return false;
             }
             AgentTerminationTask that = (AgentTerminationTask) o;
             return agentId == that.agentId &&
                timeoutMs == that.timeoutMs &&
                exitCode == that.exitCode &&
                isJvmShuttingDown == that.isJvmShuttingDown &&
                Objects.equals(appserverName, that.appserverName);
          }

          /**
           * Returns the hash of the fields uniquely identifying the object.
           *
           * @return   See above.
           */
          @Override
          public int hashCode()
          {
             return Objects.hash(appserverName, agentId, timeoutMs, exitCode, isJvmShuttingDown);
          }

          /**
           * Runs all agent termination and cleanup actions. If no tasks are currently running and no tasks are
           * queued, completes immediately the Future waiting for shutting down the sessions. Otherwise
           * registers a listener that gets notified when a session is freed and shuts it down. Starts a timer
           * to calculate the remaining time waiting for the sessions to get shutdown. If the task times out
           * the thread pool executed get forcefully shutdown. Finally the contexts for the appserver are removed
           * from the security manager and all references removed from the appserver manager.
           */
          @Override
          public void run()
          {
             if (workerPoolExecutor == null || state == State.TERMINATING || state == State.TERMINATED)
             {
                return;
             }

             outOfContextLogManager.writeMessage("Agent Shutting Down. Status: " + exitCode.getNumber(),
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_LISTENER,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             if (!isJvmShuttingDown)
             {
                OrderedShutdownHooks.deregisterShutdownHook(1, this);
             }

             boolean isForcefulTermination = timeoutMs == 0;

             ElapsedTimer timeoutTimer = new ElapsedTimer(timeoutMs);
             timeoutTimer.start();

             CompletableFuture<Boolean> shuttingDownSessionsFuture = new CompletableFuture<>();

             synchronized (sessionsLock)
             {
                if (taskQueue.size() == 0 && workerPoolExecutor.getActiveCount() == 0 && !hasAnySession())
                {
                   shuttingDownSessionsFuture.complete(true);
                }
                else
                {
                   if (isForcefulTermination)
                   {
                      taskQueue.clear();
                   }

                   terminationSessionListener = new TerminationSessionListener()
                   {
                      private AtomicBoolean isCancelled = new AtomicBoolean(false);

                      @Override
                      public void onSessionFreed(int sessionId)
                      {
                         if (!isCancelled.get())
                         {
                            shutdownSession(agentId,
                                            sessionId,
                                            agentAdminConnection,
                                            true,
                                            true,
                                            true);
                         }
                      }

                      @Override
                      public void onSessionShutdown(int sessionId)
                      {
                         if (connIdByActiveSessionId.size() == 0 && taskQueue.size() == 0)
                         {
                            shuttingDownSessionsFuture.complete(true);
                         }
                      }

                      @Override
                      public boolean isCancelled()
                      {
                         return isCancelled.get();
                      }

                      /**
                       * If not already completed, completes this CompletableFuture with a {@link CancellationException}.
                       *
                       * @param    mayInterruptIfRunning
                       *           Flag to interrupt to thread.
                       *
                       * @return   {@code true} if this task is now cancelled
                       */
                      @Override
                      public boolean cancel(boolean mayInterruptIfRunning)
                      {
                         isCancelled.set(true);
                         return true;
                      }
                   };
                }

                // should be after creating terminationSessionListener to not miss a session event
                state = State.TERMINATING;

                for (Iterator<Integer> it = idleSessions.iterator(); it.hasNext(); )
                {
                   shutdownNextSession(agentId, it, true, true, true);
                }
                for (Iterator<Integer> it = idleBoundSessions.iterator(); it.hasNext(); )
                {
                   shutdownNextSession(agentId, it, true, true, true);
                }
                for (Iterator<Integer> it = connIdByBoundSessionId.keySet().iterator(); it.hasNext(); )
                {
                   shutdownNextSession(agentId, it, true, true, true);
                }
             }

             for (SessionListenerFuture unboundSessionListener : unboundSessionListeners)
             {
                unboundSessionListener.complete(INVALID_SESSION_ID);
             }
             unboundSessionListeners.clear();

             for (SessionListenerFuture boundSessionListener : boundSessionListeners)
             {
                boundSessionListener.complete(INVALID_SESSION_ID);
             }
             boundSessionListeners.clear();

             try
             {
                long left = timeoutTimer.getLeft();
                if (LOG.isLoggable(Level.FINE))
                {
                   LOG.fine("Waiting for shutting down all sessions for the next " + left +
                               " milliseconds.");
                }
                shuttingDownSessionsFuture.get(left, TimeUnit.MILLISECONDS);

                synchronized (sessionsLock)
                {
                   if (terminationSessionListener != null)
                   {
                      terminationSessionListener.cancel(true);
                      terminationSessionListener = null;
                   }
                }
             }
             catch (Exception ignored)
             {
             }

             if (!workerPoolExecutor.isTerminated())
             {
                long left = timeoutTimer.getLeft();
                if (left == 0)
                {
                   workerPoolExecutor.shutdownNow();
                }
                else
                {
                   workerPoolExecutor.shutdown();
                   try
                   {
                      if (LOG.isLoggable(Level.FINE))
                      {
                         LOG.fine("Waiting for shutting down the worker executor for the next " + left +
                                     " milliseconds.");
                      }
                      workerPoolExecutor.awaitTermination(left, TimeUnit.MILLISECONDS);
                   }
                   catch (InterruptedException ignored)
                   {
                   }
                }
             }
             workerPoolExecutor = null;

             try
             {
                // move outside when multiple agents implemented
                msaSecurityManager.removeApp(definition.getAppserverName());
                MultiSessionAppserverManager.getInstance().clearApp(definition.getAppserverName());
                sessionCreateTime.clear();
             }
             catch (Throwable t)
             {
                LOG.warning("", t);
             }

             state = State.TERMINATED;

             outOfContextLogManager.writeMessage("Agent Shutdown Complete.",
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_LISTENER,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);
          }
       }

       /**
        * Returns <code>true</code> if there are any existing agent sessions, <code>false</code> otherwise.
        *
        * @return   See above.
        */
       private boolean hasAnySession()
       {
          synchronized (sessionsLock)
          {
             return idleSessions.size() > 0 || idleBoundSessions.size() > 0 ||
                connIdByReservedSessionId.size() > 0 || connIdByActiveSessionId.size() > 0 ||
                connIdByBoundSessionId.size() > 0;
          }
       }

       /**
        * Resets the agent session / FWD context.
        *
        * @param    agentId
        *           The agent id.
        * @param    sessionId
        *           The agent session id.
        * @param    connection
        *           The client connection.
        */
       private void resetSession(Integer agentId, Integer sessionId, MultiSessionClientConnection connection)
       {
          /* the MSA session is reset to its initial state, which includes deletion of persistent procedures and
          static MSA objects, the disconnection of databases (or re-connection if the databases were connected
          at startup), and the clean-up of all global data, such as shared variables */
          try
          {
             shutdownSession(agentId, sessionId, connection, false, false, false);
          }
          catch (Exception e)
          {
             if (LOG.isLoggable(Level.FINE))
             {
                LOG.log(Level.FINE, "Session shutdown proc throws during session reset.", e);
             }
          }

          // before cleaning up, cleanup all still-alive dynamic frames
          // TODO(with UI support): LogicalTerminal.cleanupDynamicFrames();

          // pop the global SESSION_GLOBAL_SCOPE scope, which will be pushed again in the 'startupSession' method.
          TransactionManager.popScope();

          try
          {
             msaSecurityManager.resetContext(definition.getAppserverName(), sessionId);
          }
          catch (RestrictedUseException e)
          {
             LOG.severe("RestrictedUseException in resetSession.");
          }

          try
          {
             startupSession(agentId, sessionId, connection, false, true);
          }
          catch (Exception e)
          {
             shutdownSession(agentId, sessionId, connection, true, true, false);
          }
       }

       /**
        * Finalizes the current session context by removing references and deleting the FWD context.
        *
        * @param    sessionId
        *           The agent session id.
        */
       private void finalizeCurrentContext(int sessionId)
       {
          synchronized (sessionsLock)
          {
             // should not signal the session listeners, so don't use freeSession()
             connIdByActiveSessionId.remove(sessionId);
             sessionCreateTime.remove(sessionId);
          }

          try
          {
             msaSecurityManager.deleteCurrentContext(definition.getAppserverName(), sessionId);
          }
          catch (RestrictedUseException e)
          {
             LOG.warning("", e);
          }

          if (state == State.TERMINATING && terminationSessionListener != null)
          {
             terminationSessionListener.onSessionShutdown(sessionId);
          }
       }

       /**
        * The thread to be used by the appserver thread pool executor.
        */
       private class MsaThread
       extends Thread
       {
          /** The incremental number of the thread. */
          private final int threadNumber;

          /** LOG-MANAGER instance used for logging appserver logs out of appserver context. */
          private final LegacyLogManager outOfContextLogManager;

          /**
           * Package-private constructor.
           *
           * @param    target
           *           The thread runnable.
           * @param    name
           *           The thread name.
           * @param    outOfContextLogManager
           *           LOG-MANAGER instance used for logging appserver logs out of appserver context.
           */
          MsaThread(Runnable target, String name, LegacyLogManager outOfContextLogManager)
          {
             super(target, name);
             this.threadNumber = workerNumberCounter.getAndIncrement();
             this.outOfContextLogManager = outOfContextLogManager;
          }

          /**
           * Sets the FWD context on the thread, runs the thread and creates logs for the state of the thread.
           */
          @Override
          public void run()
          {
             try
             {
                MultiSessionAppserverSecurityManager.setInitialContext();
             }
             catch (RestrictedUseException e)
             {
                LOG.warning("", e);
             }

             if (LOG.isLoggable(Level.FINEST))
             {
                LOG.finest("Starting thread " + getName());
             }

             outOfContextLogManager.writeMessage("Spawning New Worker Thread. Number: " + threadNumber,
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_LISTENER,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             super.run();

             outOfContextLogManager.writeMessage("Worker Thread exiting. Number: " + threadNumber,
                                                 LoggingEntryType.ASPlumbing,
                                                 LoggingExecEnv.AS_AUX,
                                                 LoggingSubsys.MSAS,
                                                 LoggingLevel.ERROR);

             if (LOG.isLoggable(Level.FINEST))
             {
                LOG.finest("Stopping thread " + getName());
             }
          }
       }

       /**
        * The thread factory for the appserver thread pool executor.
        */
       private class MsaThreadFactory
       implements ThreadFactory
       {
          /** The counter for the thread ids. */
          private final AtomicInteger count = new AtomicInteger(1);

          /** LOG-MANAGER instance used for logging appserver logs out of appserver context. */
          private final LegacyLogManager outOfContextLogManager;

          /**
           * Package-private constructor.
           *
           * @param    outOfContextLogManager
           *           LOG-MANAGER instance used for logging appserver logs out of appserver context.
           */
          MsaThreadFactory(LegacyLogManager outOfContextLogManager)
          {
             this.outOfContextLogManager = outOfContextLogManager;
          }

          /**
           * Creates a new thread with a MSA specific name.
           *
           * @param    runnable
           *           The runnable for the new thread.
           *
           * @return   The newly created thread.
           */
          @Override
          public Thread newThread(@Nonnull Runnable runnable)
          {
             return new MsaThread(runnable,
                                  WORKER_THREAD_NAME_PREFIX + count.getAndIncrement() +
                                       " for app " + definition.getAppserverName(),
                                  outOfContextLogManager);
          }
       }

       /**
        * An abstract class encapsulating the core MSA task functionality.
        */
       abstract class MsaTask
       implements Runnable
       {
          /** The unique task id. */
          private final int id;

          /** Flag indicating if the request is running on a bound session. */
          private final boolean bound;

          /** The connection id. */
          private final MultiSessionClientConnection connection;

          /** The agent id. */
          private final int agentId;

          /** The session id. */
          private final int sessionId;

          /** The appserver name. */
          private final String appserverName;

          /** The result for the request. */
          private final AppServerInvocationResult result;

          /** The name of the running program. */
          private final String runningProgram;

          /** Thread-safe supplier indicating if this MSA task is running. */
          private final AtomicBoolean isRunning;

          /** Thread-safe supplier indicating if the wrapper task is cancelled. */
          private final AtomicBoolean isTaskCancelled;

          /** Flag indicating if it's an initialization or finalization system task. */
          private final boolean isInitialOrFinalSetup;

          /** Stacktrace of the original thread of the task. Used for debugging. */
          private final List<StackTraceElement[]> debugStackTrace;

          /** Instance of the multi-session appserver manager. */
          private final MultiSessionAppserverManager msaManager = MultiSessionAppserverManager.getInstance();

          /** Weak reference to the executor thread this task is running on. */
          private volatile WeakReference<Thread> executorThreadRef;

          /**
           * Package-private constructor.
           *
           * @param    appserverName
           *           The appserver name.
           * @param    agentId
           *           The agent id.
           * @param    sessionId
           *           The session id.
           * @param    connection
           *           The client connection.
           * @param    result
           *           The result for the request.
           * @param    runningProgram
           *           The name of the running program.
           * @param    bound
           *           Flag indicating if the request is running on a bound session.
           * @param    isRunning
           *           Thread-safe supplier indicating if this MSA task is running.
           * @param    isTaskCancelled
           *           Thread-safe supplier indicating if the wrapper task is cancelled.
           * @param    isInitialOrFinalSetup
           *           Flag indicating if it's an initialization or finalization system task.
           */
          MsaTask(String appserverName,
                  int agentId,
                  int sessionId,
                  MultiSessionClientConnection connection,
                  AppServerInvocationResult result,
                  String runningProgram,
                  boolean bound,
                  AtomicBoolean isRunning,
                  AtomicBoolean isTaskCancelled,
                  boolean isInitialOrFinalSetup)
          {
             this.id = msaTaskIdGenerator.getAndIncrement();
             this.appserverName = appserverName;
             this.agentId = agentId;
             this.sessionId = sessionId;
             this.connection = connection;
             this.bound = bound;
             this.result = result;
             this.runningProgram = runningProgram;
             this.isRunning = isRunning;
             this.isTaskCancelled = isTaskCancelled;
             this.isInitialOrFinalSetup = isInitialOrFinalSetup;

             if (!definition.isDebugMsa())
             {
                this.debugStackTrace = null;
             }
             else
             {
                this.debugStackTrace = new ArrayList<>();
                List<StackTraceElement[]> parentTaskDebugStackTrace = msaManager.getDebugStackTrace();
                if (parentTaskDebugStackTrace != null)
                {
                   this.debugStackTrace.addAll(parentTaskDebugStackTrace);
                }
                this.debugStackTrace.add(Thread.currentThread().getStackTrace());
             }
          }

          /**
           * Runs the implementation specific core behavior.
           */
          abstract void coreRun();

          /**
           * Prepares the task and runs the implementation specific actions via {@link #coreRun()}. If the
           * wrapper task has been cancelled before the MsaTask is started by the executor, then it doesn't run.
           * Otherwise a weak reference to the executor thread is created, the {@link #isRunning} flag is set
           * to true, the FWD context is set on the thread, the agent session state is set to active, context
           * local vars are set in the appserver manager to allow access to the current session state from the
           * executing code, the currently running task and program are set to the connection and the core task
           * is run. Generic MSA error handling, logging and cleanup are performed. In case this task is a
           * nested task (started by another MsaTask) the original context is stored before coreRun is called,
           * and on completion restored to allow the original task completing in the right context.
           */
          @Override
          public final void run()
          {
             if (isTaskCancelled != null && isTaskCancelled.get())
             {
                return;
             }

             executorThreadRef = new WeakReference<>(Thread.currentThread());

             if (isRunning != null)
             {
                isRunning.set(true);
             }

             MultiSessionClientConnection originalConnection = msaManager.getConnection();
             Integer originalAgentId = msaManager.getAgentId();
             Integer originalSessionId = msaManager.getSessionId();
             boolean originalIsInitialSetup = msaManager.isInitialSetup();
             List<StackTraceElement[]> originalDebugStackTrace = msaManager.getDebugStackTrace();

             if (!switchSession(appserverName, sessionId))
             {
                executorThreadRef = null;
                outOfContextLogManager.writeMessage("Session AS-" + sessionId +
                                                       " could not be found. Cannot process request.",
                                                    LoggingEntryType.ASPlumbing,
                                                    LoggingExecEnv.AS_AUX,
                                                    LoggingSubsys.MSAS,
                                                    LoggingLevel.ERROR);
                // 1 AS-Aux-0 MSAS Worker Thread exiting. Number: 4, Status: -14
                throw new ErrorConditionException("Error handling request! Status=-1003");
             }

             useSession(connection.getId(), sessionId);

             msaManager.setMsaContext(connection, agentId, sessionId, isInitialOrFinalSetup, debugStackTrace);

             connection.setCurrentlyRunningTask(this);
             if (runningProgram != null)
             {
                connection.setCurrentlyRunningProgram(runningProgram);
             }

             if (LOG.isLoggable(Level.FINE))
             {
                LOG.fine("MsaTask #" + id + " starting on thread `" + Thread.currentThread().getName() +
                            "` for agentId: " + agentId +
                            ", sessionId: " + sessionId +
                            ", connectionId: " + connection.getId());
             }

             boolean isQuit = false;
             try
             {
                coreRun();
                if (result != null && result.getError() instanceof QuitConditionException)
                {
                   isQuit = true;
                   msaManager.resetMsaContext();
                   resetSession(agentId, this.sessionId, connection);
                }
             }
             catch (QuitConditionException e)
             {
                isQuit = true;
                msaManager.resetMsaContext();
                resetSession(agentId, this.sessionId, connection);
                throw e;
             }
             finally
             {
                if (!isQuit)
                {
                   msaManager.resetMsaContext();
                }
                executorThreadRef = null;

                connection.setCurrentlyRunningTask(null);
                connection.setCurrentlyRunningProgram(null);

                if (LOG.isLoggable(Level.FINE))
                {
                   LOG.fine("MsaTask #" + id + " ends for agentId: " + agentId +
                               ", sessionId: " + this.sessionId +
                               ", connectionId: " + connection.getId());
                }

                if (originalSessionId != null)
                {
                   switchSession(appserverName, originalSessionId);
                   msaManager.setMsaContext(originalConnection,
                                            originalAgentId,
                                            originalSessionId,
                                            originalIsInitialSetup,
                                            originalDebugStackTrace);
                }
             }
          }

          /**
           * Returns the string representation.
           *
           * @return   See above.
           */
          @Override
          public String toString()
          {
             return "MsaTask{" +
                "bound=" + bound +
                ", sessionId=" + sessionId +
                ", connectionId='" + connection.getId() + '\'' +
                '}';
          }

          /**
           * Returns the task id.
           *
           * @return   See above.
           */
          public int getId()
          {
             return id;
          }

          /**
           * Returns the thread this task is currently running on. Can be <code>null</code> if the task hasn't
           * been started or is already completed.
           *
           * @return   See above.
           */
          Thread getRunningThread()
          {
             if (executorThreadRef == null)
             {
                return null;
             }
             return executorThreadRef.get();
          }

          /**
           * Sends STOP by cancelling the task and interrupting the running thread.
           */
          void sendStop()
          {
             Thread thread = executorThreadRef.get();
             if (thread != null)
             {
                thread.interrupt();
             }
          }
       }

       /**
        * Interface of a listener listening for freed sessions during agent termination.
        */
       private interface TerminationSessionListener
       {
          /**
           * Called when a session is freed.
           *
           * @param    sessionId
           *           The session id.
           */
          void onSessionFreed(int sessionId);

          /**
           * Called when a session has been shutdown.
           *
           * @param    sessionId
           *           The session id.
           */
          void onSessionShutdown(int sessionId);

          /**
           * If not already completed, completes this CompletableFuture with a {@link CancellationException}.
           *
           * @param    mayInterruptIfRunning
           *           Flag to interrupt to thread.
           *
           * @return   {@code true} if this listener is now cancelled
           */
          boolean cancel(boolean mayInterruptIfRunning);

          /**
           * Check if the listener is cancelled.
           *
           * @return   {@code true} if this listener is cancelled
           */
          boolean isCancelled();
       }

       /**
        * Interface of a listener listening for freed sessions.
        */
       private interface SessionListener
       {
          /**
           * Called when a session is freed.
           *
           * @param    sessionId
           *           The session id.
           */
          void onSessionFreed(int sessionId);

          /**
           * Returns the id of session requested to be reserved. <code>null</code> when the connection is not
           * bound.
           *
           * @return   See above.
           */
          Integer getRequestedSessionId();

          /**
           * Check if the listener is cancelled.
           *
           * @return   {@code true} if this listener is cancelled
           */
          boolean isCancelled();
       }

       /**
        * Future completing when the session it listens for is free. Returns the session id on completion.
        */
       private class SessionListenerFuture
       extends CompletableFuture<Integer>
       implements SessionListener
       {
          /** A unique id for the instance. */
          private final String uniqueId = UUID.randomUUID().toString();

          /** The id of the connection that wants to reserve the session. */
          private final String connectionId;

          /** The id of session requested to be reserved. <code>null</code> when the connection is not bound. */
          private final Integer requestedSessionId;

          /**
           * Package-private constructor.
           *
           * @param    connectionId
           *           The id of the connection that wants to reserve the session.
           * @param    sessionId
           *           The id of session requested to be reserved. <code>null</code> when the connection is not
           *           bound.
           */
          SessionListenerFuture(String connectionId, Integer sessionId)
          {
             this.connectionId = connectionId;
             this.requestedSessionId = sessionId;
          }

          /**
           * If not already completed, completes this CompletableFuture with a {@link CancellationException}.
           *
           * @param    mayInterruptIfRunning
           *           Flag to interrupt to thread.
           *
           * @return   {@code true} if this task is now cancelled
           */
          @Override
          public boolean cancel(boolean mayInterruptIfRunning)
          {
             synchronized (sessionsLock)
             {
                complete(INVALID_SESSION_ID);

                synchronized (sessionsLock)
                {
                   unboundSessionListeners.remove(this);
                   boundSessionListeners.remove(this);
                }

                return super.cancel(mayInterruptIfRunning);
             }
          }

          /**
           * Called when a session that meets requirements gets free. This completes the Future.
           *
           * @param    sessionId
           *           The id of session that is freed.
           */
          @Override
          public void onSessionFreed(int sessionId)
          {
             reserveSession(connectionId, sessionId);
             complete(sessionId);
          }

          /**
           *
           * @return   The id of the session that is requested.
           */
          @Override
          public Integer getRequestedSessionId()
          {
             return requestedSessionId;
          }

          /**
           * Checks if the argument object represents the same listener, identified by the connectionId and
           * sessionId.
           *
           * @param    o
           *           The object to compare to.
           *
           * @return   <code>true</code> if it's the same instance or the connectionId and sessionId are the same.
           *           <code>false</code> otherwise.
           */
          @Override
          public boolean equals(Object o)
          {
             if (this == o)
             {
                return true;
             }
             if (o == null || getClass() != o.getClass())
             {
                return false;
             }
             SessionListenerFuture that = (SessionListenerFuture) o;
             return Objects.equals(uniqueId, that.uniqueId);
          }

          /**
           * Returns the hash of the fields uniquely identifying the object.
           *
           * @return   See above.
           */
          @Override
          public int hashCode()
          {
             return Objects.hash(uniqueId);
          }
       }
    }