MultiSessionAppserver.java

/*
** Module   : MultiSessionAppserver.java
** Abstract : Appserver with multi-session agents.
**
** Copyright (c) 2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------Description----------------------------------
** 001 GBB 20250122 Created initial version.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
**
**   0. Attribution Requirement.
**
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
**
**   1. No License To Use Trademarks.
**
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
**
**   2. No Misrepresentation of Affiliation.
**
**     You may not represent yourself as Golden Code Development Corporation or FWD.
**
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
**
**   3. No Misrepresentation of Source or Origin.
**
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util.appserver;

import com.goldencode.p2j.admin.*;
import com.goldencode.p2j.main.*;
import com.goldencode.p2j.oo.lang.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.util.*;
import com.goldencode.p2j.util.logging.*;

import javax.annotation.*;
import java.lang.ref.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.function.*;
import java.util.logging.*;

/** Appserver with multi-session agents. */
class MultiSessionAppserver
{
   /** Enum for the state of the multi-session appserver */
   enum State
   {
      INITIALIZING,
      RUNNING,
      TERMINATING,
      TERMINATED
   }
   
   /** ID of invalid id, used to indicate no session found. */
   static final int INVALID_SESSION_ID = -1;
   
   /** Logger. */
   private static final CentralLogger LOG = CentralLogger.get(MultiSessionAppserver.class);

   /** The Future for reserving a session, when the result is an invalid session. */
   private static final CompletableFuture<Integer> NO_SESSION_FOUND_RESULT =
      CompletableFuture.completedFuture(INVALID_SESSION_ID);
   
   /** The name prefix for the worker threads. */
   private static final String WORKER_THREAD_NAME_PREFIX = "MSA Worker #";
   
   /** The session global scope */
   private static final String SESSION_GLOBAL_SCOPE = "msa-session";
   
   /**
    * Undocumented OE config 'minAvailableABLSessions', min number of MSA sessions is always equal or bigger
    * than 1.
    */
   private static final int MIN_ABL_SESSIONS = 1;
   
   /** Min number of worker threads at any time. */
   private static final int MIN_CONNECTIONS = 1;

   /** ID generator for the MSA tasks. */
   private static final AtomicInteger msaTaskIdGenerator = new AtomicInteger(1);

   /** The definition of the current appserver. */
   private final MultiSessionAppserverDefinition definition;

   /** The agent connection for the system tasks. */
   private final MultiSessionClientConnection agentAdminConnection;

   /** Counter for the incremental id of the worker threads. */
   private final AtomicInteger workerNumberCounter = new AtomicInteger(1);

   /** The idle resources watchdog. Kept as a field to prevent being GCed. */
   private final ScheduledExecutorService watchdogExecutor =
      Executors.newSingleThreadScheduledExecutor(r -> 
      {
         Thread newThread = new Thread(r, "MSA Idle Resources Watchdog");
         newThread.setDaemon(true);
         return newThread;
      });

   /** The startup time for the current appserver. */
   private final Date startupTime;

   /** Sorted set of available unbound sessions in idle state. */
   private final TreeSet<Integer> idleSessions = new TreeSet<>();

   /** Sorted set of available bound sessions in idle state. */
   private final Set<Integer> idleBoundSessions = new HashSet<>();
   
   /** Reserved session id : connection id pairs. */
   private final Map<Integer, String> connIdByReservedSessionId = new HashMap<>();

   /** Bound session id : connection id pairs. Bound connections can be idle/reserved/active. */
   private final Map<Integer, String> connIdByBoundSessionId = new HashMap<>();

   /** Active session id : connection id pairs. */
   private final Map<Integer, String> connIdByActiveSessionId = new HashMap<>();

   /** Map of session ids and their creation dates. */
   private final Map<Integer, Date> sessionCreateTime = new ConcurrentHashMap<>();

   /** List of listeners waiting for a free unbound session. */
   private final List<SessionListenerFuture> unboundSessionListeners = new ArrayList<>();
   
   /** List of listeners waiting for a free bound session. */
   private final List<SessionListenerFuture> boundSessionListeners = new ArrayList<>();

   /** The lock to synchronize any agent session operations */
   private final Object sessionsLock = new Object();

   /** The security manager for the multi-session agent appservers. */
   private final MultiSessionAppserverSecurityManager msaSecurityManager = 
      MultiSessionAppserverSecurityManager.getInstance();

   /** LOG-MANAGER instance used for logging appserver logs out of appserver context. */
   private LegacyLogManager outOfContextLogManager;

   /** The startup parameters for the agent. */
   private StartupParameters startupParameters;

   /** The executor for the worker thread pool. */
   private ThreadPoolExecutor workerPoolExecutor;

   /** The queued tasks to be running in the {@link #workerPoolExecutor}. */
   private LinkedBlockingQueue<Runnable> taskQueue;

   /** Listener for freed sessions, receiving calls during agent termination. */
   private TerminationSessionListener terminationSessionListener;

   /** The EXPORTS for this appserver. */
   private volatile String exports;

   /** The current state of the appserver. */
   private volatile State state;

   /**
    * Package-private constructor.
    * 
    * @param    definition
    *           The definition for the appserver to be started.
    * @param    connectionsSupplier
    *           Supplier for all currently active connections.
    * @param    disconnectMultipleFunc
    *           Function to disconnect connections, provided in a collection.
    * 
    * @throws   Exception
    *           Any exception thrown during starting the appserver.
    */
   MultiSessionAppserver(MultiSessionAppserverDefinition definition,
                         Supplier<Collection<MultiSessionClientConnection>> connectionsSupplier,
                         Consumer<Collection<MultiSessionClientConnection>> disconnectMultipleFunc)
   throws Exception
   {
      this.definition = definition;
      this.state = State.INITIALIZING;
      this.agentAdminConnection = new MultiSessionClientConnection(MultiSessionClientConnection.AGENT_SYS_CONN_ID,
                                                                   null,
                                                                   definition.getAppserverName(),
                                                                   false,
                                                                   false);
      
      prepareLogging();

      AgentTerminationTask terminationTask = new AgentTerminationTask(MultiSessionAgent.SINGLE_AGENT_ID,
                                                                      definition.getCompleteActiveReqTimeout(),
                                                                      MultiSessionAgent.ExitCode.EXPECTED_4020,
                                                                      true);
      OrderedShutdownHooks.registerShutdownHook(1, terminationTask);
      
      initializeAgent();
      
      this.state = State.RUNNING;
      this.startupTime = new Date();

      int watchdogPeriod = definition.getIdleResourceTimeout();
      int clientConnectionTimeout = definition.getIdleSessionTimeout();
      if (watchdogPeriod > 0 && clientConnectionTimeout > 0)
      {
         watchdogExecutor.scheduleAtFixedRate(new Runnable()
         {
            String appserverName = definition.getAppserverName();

            @Override
            public void run()
            {
               if (LOG.isLoggable(Level.FINEST))
               {
                  LOG.finest("Running MSA idle resources watchdog.");
               }

               long now = new Date().getTime();

               // mark client connections for removal
               List<MultiSessionClientConnection> connectionsToDisconnect = new ArrayList<>();
               for (MultiSessionClientConnection connection : connectionsSupplier.get())
               {
                  if (connection.getAppserverName().equals(appserverName) &&
                     !connection.getId().equals(MultiSessionClientConnection.AGENT_SYS_CONN_ID) &&
                     !connection.isForWebService() &&
                     (connection.getParentConnection() == null || 
                        !connection.getParentConnection().isForWebService()) &&
                     !connection.isInUse() &&
                     !connection.isDisconnecting() &&
                     connection.getLastUseTime().getTime() + clientConnectionTimeout < now)
                  {
                     connectionsToDisconnect.add(connection);
                  }
               }

               if (connectionsToDisconnect.size() > 0 && LOG.isLoggable(Level.FINE))
               {
                  LOG.fine("Idle connections: " + connectionsToDisconnect.size());
               }

               disconnectMultipleFunc.accept(connectionsToDisconnect);
            }
         }, watchdogPeriod, watchdogPeriod, TimeUnit.MILLISECONDS);
      }
   }

   /**
    * Returns the definition for the appserver.
    * 
    * @return   See above.
    */
   MultiSessionAppserverDefinition getDef()
   {
      return definition;
   }

   /**
    * Returns the number of running agents.
    * 
    * @return   See above.
    */
   int getNumberOfAgents()
   {
      return 1;
   }

   /**
    * Returns the number of the started sessions (idle, reserved and active).
    *
    * @return   See above.
    */
   int getNumberOfSessions()
   {
      synchronized (sessionsLock)
      {
         return idleSessions.size() + idleBoundSessions.size() + 
            connIdByReservedSessionId.size() + connIdByActiveSessionId.size();
      }
   }
   
   /**
    * Returns the time of the appserver start.
    *
    * @return   See above.
    */
   Date getStartupTime()
   {
      return startupTime;
   }

   /**
    * Returns the client connection used for system tasks.
    *
    * @return   See above.
    */
   MultiSessionClientConnection getAdminConnection()
   {
      return agentAdminConnection;
   }
   
   /**
    * Get the exports for this appserver.  If set, only the external procedures which match these
    * exports will be available for invocation.
    *
    * @return   See above.
    */
   character getExports()
   {
      return exports == null ? null : new character(exports);
   }

   /**
    * Set the exports for this appserver.
    *
    * @param    exports
    *           The appserver's exports. 
    *           
    * @return   <code>true</code> if the method succeeded.
    */
   logical setExports(character exports)
   {
      this.exports = (exports.isUnknown() ? null : exports.toStringMessage());
      
      return new logical(true);
   }

   /**
    * Returns an array of descriptions for all started sessions (idle, reserved and active).
    *
    * @return   See above.
    */
   MultiSessionAgentSessionDef[] getSessionDefs()
   {
      List<MultiSessionAgentSessionDef> sessionInfos = new ArrayList<>();
      String appserverName = definition.getAppserverName();
      MultiSessionAppserverManager msaManager = MultiSessionAppserverManager.getInstance();

      synchronized (sessionsLock)
      {
         for (int idleSessionId : idleSessions)
         {
            sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                             MultiSessionAgent.SINGLE_AGENT_ID,
                                                             idleSessionId,
                                                             sessionCreateTime.get(idleSessionId),
                                                             false,
                                                             MultiSessionAgentSessionDef.State.IDLE,
                                                             null,
                                                             null,
                                                             null));
         }

         for (int idleBoundSessionId : idleBoundSessions)
         {
            MultiSessionClientConnection connection = 
               msaManager.getConnection(connIdByBoundSessionId.get(idleBoundSessionId));
            if (connection == null)
            {
               continue;
            }
            sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                             MultiSessionAgent.SINGLE_AGENT_ID,
                                                             idleBoundSessionId,
                                                             sessionCreateTime.get(idleBoundSessionId),
                                                             true,
                                                             MultiSessionAgentSessionDef.State.IDLE,
                                                             connection.getModel().toString(),
                                                             null,
                                                             null));
         }
         
         for (int reservedSessionId : connIdByReservedSessionId.keySet())
         {
            MultiSessionClientConnection connection =
               msaManager.getConnection(connIdByReservedSessionId.get(reservedSessionId));
            if (connection == null)
            {
               continue;
            }
            sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                             MultiSessionAgent.SINGLE_AGENT_ID,
                                                             reservedSessionId,
                                                             sessionCreateTime.get(reservedSessionId),
                                                             connIdByBoundSessionId.containsKey(reservedSessionId),
                                                             MultiSessionAgentSessionDef.State.RESERVED,
                                                             connection.getModel().toString(),
                                                             null,
                                                             null));
         }

         for (int activeSessionId : connIdByActiveSessionId.keySet())
         {
            MultiSessionClientConnection connection =
               msaManager.getConnection(connIdByActiveSessionId.get(activeSessionId));
            if (connection == null)
            {
               continue;
            }
            sessionInfos.add(new MultiSessionAgentSessionDef(appserverName,
                                                             MultiSessionAgent.SINGLE_AGENT_ID,
                                                             activeSessionId,
                                                             sessionCreateTime.get(activeSessionId),
                                                             connIdByBoundSessionId.containsKey(activeSessionId),
                                                             MultiSessionAgentSessionDef.State.ACTIVE,
                                                             connection.getModel().toString(),
                                                             connection.getCurrentlyRunningProgram(),
                                                             connection.getCurrentlyRunningProgramStartTime()));
         }
      }
      
      return sessionInfos.toArray(new MultiSessionAgentSessionDef[0]);
   }

   /**
    * Binds the client connection to the agent session if the session is not bound already. In Session-free 
    * only persistent procedures are bound (to a child connection).
    * 
    * @param    connection
    *           The client connection to get bound.
    * @param    sessionId
    *           The agent session id to get bound.
    * @param    isPersistentFunc
    *           True if a persistent function is initiating the bound.
    */
   void bind(MultiSessionClientConnection connection, Integer sessionId, boolean isPersistentFunc)
   {
      if ((connection.isSessionFree() && !isPersistentFunc) || connIdByBoundSessionId.containsKey(sessionId))
      {
         return;
      }
      synchronized (sessionsLock)
      {
         connIdByBoundSessionId.put(sessionId, connection.getId());
      }
      connection.setBoundSessionId(sessionId);
   }

   /**
    * Unbinds the client connection from the agent session if it's already bound. In Session-free only 
    * persistent procedures are bound (to a child connection).
    *
    * @param    connection
    *           The client connection to get bound.
    * @param    sessionId
    *           The agent session id to get bound.
    * @param    isPersistentFunc
    *           True if a persistent function is initiating the bound.
    */
   void unbind(MultiSessionClientConnection connection, Integer sessionId, boolean isPersistentFunc)
   {
      if ((connection.isSessionFree() && !isPersistentFunc) || !connection.isBound())
      {
         return;
      }
      synchronized (sessionsLock)
      {
         String foundConnectionId = connIdByBoundSessionId.remove(sessionId);
         if (!foundConnectionId.equals(connection.getId()))
         {
            // indicates bug in the logic
            LOG.warning("Mismatched session removed from bound connection.");
         }
      }
      connection.setBoundSessionId(null);
      // TODO: delete all remote persistent procedures running in the session
   }

   /**
    * Returns <code>true</code> if the appserver is running.
    * 
    * @return   See above.
    */
   boolean isRunning()
   {
      return state == State.RUNNING;
   }

   /**
    * Returns a Future for the reserved session id. Bound connections reserve the respective bound session,
    * and if it's already in use, a session listener is registered waiting it to be freed. The Future is 
    * immediately returned and completed on reserving the freed session. For unbound connections the first 
    * session in the idle sessions collection is reserved. If no sessions are idle, a new session is 
    * created. If the max sessions number is reached, a session listener is registered waiting for any 
    * unbound session to get freed.
    * 
    * @param    connection
    *           The client connection.
    * 
    * @return   Future for the session id.
    */
   Future<Integer> reserveSession(MultiSessionClientConnection connection)
   {
      if (state == State.TERMINATED || state == State.TERMINATING)
      {
         return NO_SESSION_FOUND_RESULT;
      }
      
      String connectionId = connection.getId();
      Integer boundSessionId = connection.getBoundSessionId();
         
      synchronized (sessionsLock)
      {
         if (boundSessionId != null)
         {
            if (!connIdByBoundSessionId.containsKey(boundSessionId))
            {
               // the MSA has been manually or auto removed
               LOG.warning("MSA session not found for bound connection.");
               return NO_SESSION_FOUND_RESULT;
            }
            
            if (idleBoundSessions.contains(boundSessionId))
            {
               reserveSession(connectionId, boundSessionId);
               return CompletableFuture.completedFuture(boundSessionId);
            }
           
            if (!connIdByReservedSessionId
               .containsKey(boundSessionId) && !connIdByActiveSessionId.containsKey(boundSessionId))
            {
               // indicates bug in the logic
               LOG.warning("MSA session not found for bound connection.");
               return NO_SESSION_FOUND_RESULT;
            }

            return registerSessionListener(connectionId, boundSessionId);
         }
         
         if (idleSessions.size() > 0)
         {
            Integer idleSessionId = idleSessions.first();
            reserveSession(connectionId, idleSessionId);
            return CompletableFuture.completedFuture(idleSessionId);
         }
         
         if (connIdByActiveSessionId.size() + connIdByReservedSessionId.size() + idleSessions.size() + 
            idleBoundSessions.size() >= definition.getMaxMsaSessionsPerAgent())
         {
            LOG.warning("The max number of MSA sessions is reached. Waiting for a session to get freed.");
            return registerSessionListener(connectionId, null);
         }
      }

      int newSessionId = createSession();
      try
      {
         MsaTask task = createSessionStartupTask(MultiSessionAgent.SINGLE_AGENT_ID,
                                                 newSessionId,
                                                 agentAdminConnection,
                                                 false,
                                                 false);
         submitTask(task).get();
         synchronized (sessionsLock)
         {
            // don't use freeSession() and reserveSession(), 
            // because the session can be snatched by another request
            connIdByActiveSessionId.remove(newSessionId);
            connIdByReservedSessionId.put(newSessionId, connectionId);
         }
         return CompletableFuture.completedFuture(newSessionId);
      }
      catch (Exception e)
      {
         AppServerOperatingMode mode = connection.isSessionFree() ? 
            AppServerOperatingMode.STATE_FREE :
            AppServerOperatingMode.STATELESS;
         String log = "Could not get and/or initialize '" + mode + "' session! Cannot process request.";
         outOfContextLogManager.writeMessage(log,
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_AUX,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);

         outOfContextLogManager.writeMessage("Error handling request! Status=-1003",
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_AUX,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);

         shutdownSession(MultiSessionAgent.SINGLE_AGENT_ID, newSessionId);
         return NO_SESSION_FOUND_RESULT;
      }
   }

   /**
    * Wraps a generic execution of a request received as a Runnable. A new MsaTask is created, submitted to
    * the executor and finally the session is freed. The MsaTask wraps starting the Runnable in a common
    * code that manages a top 'startup' scope, sets CURRENT-REQUEST-INFO, sets the return value or error as
    * the result.
    * 
    * @param    connection
    *           The client connection.
    * @param    sessionId
    *           The session id.
    * @param    invokeRunnable
    *           The runnable to be started.
    * 
    * @throws   Exception
    *           Any exception occurring during execution.
    */
   void runInTopScope(MultiSessionClientConnection connection, int sessionId, Runnable invokeRunnable)
   throws Exception
   {
      runInTopScope(null, null, connection, sessionId, null, invokeRunnable, null, null);
   }

   /**
    * Wraps a generic execution of a request received as a Runnable. A new MsaTask is created, submitted to
    * the executor and finally the session is freed. The MsaTask wraps starting the Runnable in a common 
    * code that manages a top 'startup' scope, sets CURRENT-REQUEST-INFO, sets the return value or error as
    * the result.
    *
    * @param    result
    *           The result of the appserver call.
    * @param    requestInfo
    *           The request info data.
    * @param    connection
    *           The client connection.
    * @param    sessionId
    *           The session id.
    * @param    runningProgram
    *           The name of the program to be executed.
    * @param    invokeRunnable
    *           The runnable to be started.
    * @param    isTaskRunning
    *           Thread-safe supplier indicating if the wrapper task is running.
    * @param    isTaskCancelled
    *           Thread-safe supplier indicating if the wrapper task is cancelled.
    *
    * @throws   Exception
    *           Any exception occurring during execution.
    */
   void runInTopScope(AppServerInvocationResult result,
                      Object[] requestInfo,
                      MultiSessionClientConnection connection,
                      int sessionId,
                      String runningProgram,
                      Runnable invokeRunnable,
                      AtomicBoolean isTaskRunning,
                      AtomicBoolean isTaskCancelled)
   throws Exception
   {
      MsaTask task = new MsaTask(definition.getAppserverName(),
                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                 sessionId,
                                 connection,
                                 result,
                                 runningProgram,
                                 isBound(connection.getId()),
                                 isTaskRunning,
                                 isTaskCancelled,
                                 false)
      {
         @Override
         public void coreRun()
         {
            // add the topmost scope to the TransactionManager
            TransactionManager.pushScope("startup",
                                         TransactionManager.NO_TRANSACTION,
                                         true,
                                         true,
                                         false,
                                         false);

            if (requestInfo != null && requestInfo.length > 0)
            {
               SessionUtils.currentRequestInfo().ref().fromArray(requestInfo);
            }

            try
            {
               invokeRunnable.run();
            }
            catch (Throwable t)
            {
               if (!UnstoppableExitException.isCausing(t))
               {
                  LOG.log(Level.WARNING,
                          t,
                          "runInTopScope has failed for appserver %s:",
                          definition.getAppserverName());
                  if (result != null)
                  {
                     result.setError(t);
                  }
               }
            }
            finally
            {
               boolean isContextDeleted = TransactionManager.getScopeLabel() == null;
               if (!isContextDeleted)
               {
                  // remove the topmost scope from the TransactionManager
                  TransactionManager.popScope();
               }
            }

            if (ControlFlowOps.isReturnValueSetInRootScope() && result != null)
            {
               result.setResult(ControlFlowOps.getReturnValue());
            }
         }
      };
      
      try
      {
         submitTask(task).get();
      }
      finally
      {
         freeSession(sessionId, true);
      }
   }

   /**
    * A simple wrapper for a generic execution of a request in the PASOE context. A new MsaTask is created, 
    * submitted to the executor and finally the session is freed.
    * 
    * @param    connection
    *           The client connection.
    * @param    sessionId
    *           The session id.
    * @param    result
    *           The result of the appserver call.
    * @param    runnable
    *           The runnable to be started.
    * @param    isTaskRunning
    *           Thread-safe supplier indicating if the MSA task is running.
    * @param    isTaskCancelled
    *           Thread-safe supplier indicating if the wrapper task is cancelled.
    * @param    taskReceiver
    *           Function called to supply the newly created MsaTask to the caller.
    * 
    * @throws   Exception
    *           Any exception occurring during execution.
    */
   void runInSessionContext(MultiSessionClientConnection connection,
                            int sessionId,
                            AppServerInvocationResult result,
                            Runnable runnable,
                            AtomicBoolean isTaskRunning,
                            AtomicBoolean isTaskCancelled,
                            Function<MsaTask, Void> taskReceiver)
   throws Exception
   {
      MsaTask task = new MsaTask(definition.getAppserverName(),
                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                 sessionId,
                                 connection,
                                 result,
                                 null,
                                 isBound(connection.getId()),
                                 isTaskRunning,
                                 isTaskCancelled,
                                 false)
      {
         @Override
         void coreRun()
         {
            runnable.run();
         }
      };
      
      if (taskReceiver != null)
      {
         taskReceiver.apply(task);
      }

      try
      {
         submitTask(task).get();
      }
      finally
      {
         freeSession(sessionId, true);
      }
   }

   /**
    * Terminate the appserver by running the termination tasks for all agents.
    * 
    * @param    exitCode
    *           The exit code for the appserver.
    */
   void terminateAppserver(MultiSessionAgent.ExitCode exitCode)
   {
      terminateAgent(MultiSessionAgent.SINGLE_AGENT_ID, exitCode);
   }

   /**
    * Terminate the agent by running the termination task.
    *
    * @param    exitCode
    *           The exit code for the appserver.
    */
   void terminateAgent(int agentId, MultiSessionAgent.ExitCode exitCode)
   {
      // If multiple agents are implemented sweeping an idle agent should set the timeout to 
      // definition.getDefaultAgentWaitToFinish() + definition.getDefaultAgentWaitAfterStop()
      new AgentTerminationTask(agentId, definition.getCompleteActiveReqTimeout(), exitCode, false).run();
   }

   /**
    * Terminate the agent session. Cleans up all references to the session. If the session is currently in
    * use, registers a session listener. Runs the session shutdown procedure. If requested, finalizes the FWD
    * security context. If it's the last session on the agent and the flag is raised, it also terminates 
    * the agent.
    * 
    * @param    agentId
    *           The agent id.
    * @param    sessionId
    *           The session id.
    * @param    finalizeCurrentContext
    *           Flag to indicate if the current context needs to be finalized after the session shut down.
    * @param    finalizeAgent
    *           Flag to indicate if the current agent needs to be terminated after the session shut down 
    *           (in effect only when the last session in the agent is shut down).
    * 
    * @return   Future that gets completed when the termination of the session is done.
    */
   Future<?> terminateSession(int agentId,
                              int sessionId,
                              boolean finalizeCurrentContext,
                              boolean finalizeAgent)
   {
      Future<Integer> freeSessionFuture = null;
      
      synchronized (sessionsLock)
      {
         boundSessionListeners.removeIf(l -> l.requestedSessionId == sessionId);
         
         if (connIdByActiveSessionId.containsKey(sessionId))
         {
            String connectionId = connIdByActiveSessionId.get(sessionId);
            if (connectionId != null)
            {
               MultiSessionClientConnection connection = MultiSessionAppserverManager.getInstance()
                                                                                     .getConnection(connectionId);
               freeSessionFuture =
                  registerSessionListener(connection == null ? null : connection.getId(), sessionId);

               if (connection != null)
               {
                  connection.stopCurrentTask();
               }
            }
         }
         else
         {
            idleSessions.remove(sessionId);
            idleBoundSessions.remove(sessionId);
            connIdByReservedSessionId.remove(sessionId);
         }

         connIdByBoundSessionId.remove(sessionId);
      }
      
      if (freeSessionFuture != null)
      {
         try
         {
            // TODO; timeout?
            freeSessionFuture.get();
         }
         catch (Exception ignored)
         {
         }
      }
      
      try
      {
         return shutdownSession(agentId,
                                sessionId,
                                agentAdminConnection,
                                finalizeCurrentContext,
                                finalizeAgent, 
                                false);
      }
      finally
      {
         freeSession(sessionId, false, true);
      }
   }
   
   /**
    * Delete the remote procedures active on the session and connection. This must be called at the same
    * time when the local proxy procedure is deleted.
    *
    * @param    connection
    *           The client connection.
    * @param    shutdownBoundSession
    *           Flag to indicate the bound MSA session should be shut down.
    */
   void deleteConnection(MultiSessionClientConnection connection, boolean shutdownBoundSession)
   {
      boolean hasPersistentProc = connection.hasPersistentProc();
      for (AppserverRemoteProcedure remote : connection.getPersistentProcs())
      {
         Object referent = remote.getExtProg().get();
         if (referent instanceof _BaseObject_)
         {
            connection.deletePersistentObject(remote.getExtProgResourceId());
         }
         else
         {
            connection.deletePersistentProc(remote.getExtProgResourceId());
         }
      }

      if (shutdownBoundSession && connection.isBound())
      {
         shutdownSession(MultiSessionAgent.SINGLE_AGENT_ID, connection.getBoundSessionId());
      }

      String connectionId = connection.getId();
      clearConnection(connectionId, shutdownBoundSession);

      if (!connection.isSessionFree() || hasPersistentProc)
      {
         log("Application Server disconnected with connection id: " + connectionId + ". (8359)",
             LoggingLevel.BASIC,
             LoggingEntryType.DB_Connects,
             LoggingExecEnv.AS_ADMIN);
      }
   }

   /**
    * Iterates over all bound connections searching to delete the specified connection by its id. The bound
    * session is removed from the collection of idle bound sessions and moved to the idle unbound sessions,
    * if the the session is not about to be shut down.
    * 
    * @param    connectionId
    *           The connection id.
    * @param    shutdownBoundSession
    *           Flag to indicate the bound MSA session is to be shut down.
    */
   void clearConnection(String connectionId, boolean shutdownBoundSession)
   {
      synchronized (sessionsLock)
      {
         for (Iterator<Map.Entry<Integer, String>> it = connIdByBoundSessionId.entrySet().iterator();
              it.hasNext();)
         {
            Map.Entry<Integer, String> entry = it.next();
            if (entry.getValue().equals(connectionId))
            {
               int boundSessionId = entry.getKey();
               idleBoundSessions.remove(boundSessionId);
               if (!shutdownBoundSession)
               {
                  idleSessions.add(boundSessionId);
               }
               it.remove();
            }
         }
      }
   }

   /**
    * Iterates over all idle sessions and removes them one by one, running each shutdown task.
    * 
    * @param    agentId
    *           The agent id.
    */
   void removeIdleSessions(int agentId)
   {
      synchronized (sessionsLock)
      {
         for (Iterator<Integer> it = idleSessions.iterator(); it.hasNext();)
         {
            shutdownNextSession(agentId, it, true, false, false);
         }
      }
   }

   /**
    * Writes a legacy log record to the appserver log file using a logger out of the appserver context.
    * 
    * @param    msg
    *           The log message.
    * @param    level
    *           The LOG-MANAGER log level.
    * @param    entryType
    *           The type of the entry.
    * @param    execEnv
    *           The LOG-MANAGER execution env.
    */
   void log(String msg, 
            LoggingLevel level,
            LoggingEntryType entryType,
            LoggingExecEnv execEnv)
   {
      outOfContextLogManager.writeMessage(msg, entryType, execEnv, LoggingSubsys.AS, level);
   }

   /**
    * Finds the LOG-MANAGER configurations for this appserver, instantiates a new logger out of the appserver 
    * context and initializes the logger.
    */
   private void prepareLogging()
   {
      // replicates AppServerManager.startAppServer
      LegacyLogManagerConfigs legacyLogManagerConfigs = 
         LegacyLogOps.appserverNameLogConfigPairs.get(definition.getAppserverName());

      outOfContextLogManager = LegacyLogOps.newStaticLogMgr();
      outOfContextLogManager.initialize(legacyLogManagerConfigs);
   }

   /**
    * Runs all necessary actions to setup a new appserver agent.
    * 
    * @throws   Exception
    *           Any exception thrown during agent initialization.
    */
   private void initializeAgent()
   throws Exception
   {
      int agentId = MultiSessionAgent.SINGLE_AGENT_ID;
      
      //TODO: what about definition.getAgentStartupParam() ?
      startupParameters = StandardServer.initializeStartupParameters(new StartupParameters());

      taskQueue = new LinkedBlockingQueue<>();
      workerPoolExecutor = new ThreadPoolExecutor(definition.getMaxConnectionsPerAgent(),
                                                  definition.getMaxConnectionsPerAgent(),
                                                  definition.getIdleConnectionTimeout(),
                                                  TimeUnit.MILLISECONDS,
                                                  taskQueue,
                                                  new MsaThreadFactory(outOfContextLogManager));
      
      workerPoolExecutor.allowCoreThreadTimeOut(true);
      workerPoolExecutor.prestartAllCoreThreads();

      int firstSessionId = createSession();
      try
      {
         startupSession(agentId, firstSessionId, agentAdminConnection, true, false);
      }
      catch (Throwable t)
      {
         outOfContextLogManager.writeMessage("Failed to get or initialize first user session.",
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_AUX,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);

         outOfContextLogManager.writeMessage("agentStartupProc Failed -- Shutting down...",
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_LISTENER,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);

         LOG.warning("Initializing the MSA agent with id " + agentId + " failed for appserver " +
                        definition.getAppserverName(), t);
         terminateAgent(agentId, MultiSessionAgent.ExitCode.UNEXPECTED_1003);

         // need to cast to Exception to comply with the FutureTask interface
         Throwable cause = t.getCause();
         if (cause != null)
         {
            throw (Exception) cause;
         }
         else
         {
            throw (Exception) t;
         }
      }

      for (int i = MIN_ABL_SESSIONS; i < definition.getNumInitialSessions(); i++)
      {
         int sessionId = createSession();
         try
         {
            startupSession(agentId, sessionId, agentAdminConnection, false, false);
         }
         catch (Throwable t)
         {
            // TODO: test if secondary sessions failure stops server init
            // need to cast to Exception to comply with the FutureTask interface
            Throwable cause = t.getCause();
            if (cause != null)
            {
               throw (Exception) cause;
            }
            else
            {
               throw (Exception) t;
            }
         }
      }
   }
   
   /**
    * Runs all necessary actions to setup the appserver session.
    *
    * @param    agentId
    *           The agent id.
    * @param    sessionId
    *           The id of the FWD context to be used for the session.
    * @param    connection
    *           The client connection.
    * @param    runAgentStartupProc
    *           Flag to indicate it's the first session in the agent and the agent startup procedure needs 
    *           to run.
    * @param    sessionReset
    *           Flag to indicate the setup is part of a session reset.
    * 
    * @throws   Exception
    *           Any exception thrown during agent initialization.
    */
   private void startupSession(int agentId, 
                               int sessionId,
                               MultiSessionClientConnection connection,
                               boolean runAgentStartupProc,
                               boolean sessionReset)
   throws Exception
   {
      MsaTask task = createSessionStartupTask(MultiSessionAgent.SINGLE_AGENT_ID,
                                              sessionId,
                                              connection,
                                              runAgentStartupProc,
                                              sessionReset);
      
      try
      {
         submitTask(task).get();
      }
      catch (Throwable t)
      {
         shutdownSession(agentId, sessionId);
         throw t;
      }
      
      freeSession(sessionId, true);
   }

   /**
    * Creates an MsaTask running all necessary actions to setup the session.
    * 
    * @param    agentId
    *           The agent id.
    * @param    sessionId
    *           The id of the FWD context to be used for the session.
    * @param    connection
    *           The client connection.
    * @param    runAgentStartupProc
    *           Flag to indicate it's the first session in the agent and the agent startup procedure needs 
    *           to run.
    * @param    sessionReset
    *           Flag to indicate the setup is part of a session reset.
    *           
    * @return   The MsaTask to run the session setup.
    */
   private MsaTask createSessionStartupTask(int agentId,
                                            int sessionId,
                                            MultiSessionClientConnection connection,
                                            boolean runAgentStartupProc,
                                            boolean sessionReset)
   {
      return new MsaTask(definition.getAppserverName(),
                         agentId,
                         sessionId,
                         connection,
                         null,
                         null,
                         false,
                         null,
                         null,
                         MultiSessionClientConnection.AGENT_SYS_CONN_ID.equals(connection.getId()))
      {
         @Override
         public void coreRun()
         {
            if (!sessionReset)
            {
               String msg = "Starting MSAS Session for " + definition.getAppserverName() + ".";
               outOfContextLogManager.writeMessage(msg,
                                                   LoggingEntryType.ASPlumbing,
                                                   null,
                                                   LoggingSubsys.AS,
                                                   LoggingLevel.BASIC);
            }
            
            // replicates AgentPool.start
            Utils.setProjectToken(definition.getProjectToken());

            // resolve clientConfig/cfgOverrides
            ConfigItem.resolveCfgOverrides(Utils.DirScope.BOTH, definition.getAccountIds());

            // replicates ClientCore.initialize
            ClientParameters clientParameters = definition.getClientParameters();
            clientParameters.socketTrustStoreFilename = ConfigItem.SOCKET_TRUSTSTORE_FILE
               .read("trusted-cert.store");
            clientParameters.socketTrustStorePassword = ConfigItem.SOCKET_TRUSTSTORE_PASS.read("");

            StandardServer.setClientParams(clientParameters);

            // replicates StandardServer.initializeStartupParameters
            SessionUtils.setStartupParameters(startupParameters);

            // replicates StandardServer.setupSession
            SessionUtils.setDeviceId(clientParameters.osDeviceId);
            SessionUtils.setClientTimezone(clientParameters.clientTimezone);

            // replicates Agent.prepare
            CoreAppserver.prepareSession(true, 
                                         definition.getAppserverName(), 
                                         definition.getPropath(), 
                                         SESSION_GLOBAL_SCOPE);

            String agentStartupProc = definition.getAgentStartupProc();
            String sessionStartupProc = definition.getSessionStartupProc();
            
            if (!sessionReset)
            {
               outOfContextLogManager.writeMessage("MSAS Session Startup. (5473)",
                                                   LoggingEntryType.ASPlumbing,
                                                   null,
                                                   LoggingSubsys.AS,
                                                   LoggingLevel.BASIC);
            }
            
            if ((!runAgentStartupProc || agentStartupProc == null) && sessionStartupProc == null)
            {
               // no startup procs, initialization completed
               return;
            }

            // add the topmost scope to the TransactionManager
            TransactionManager.pushScope("startup",
                                         TransactionManager.NO_TRANSACTION,
                                         true,
                                         true,
                                         false,
                                         false);

            if (runAgentStartupProc && agentStartupProc != null)
            {
               runStartupProc(true, agentStartupProc, definition.getAgentStartupProcParam());
            }
            
            if (sessionStartupProc != null)
            {
               runStartupProc(false, sessionStartupProc, definition.getSessionStartupProcParam());
            }

            TransactionManager.popScope(); // remove the "startup" scope
         }
      };
   }

   /**
    * Runs the agent or session startup procedure.
    * 
    * @param    isAgentStartupProc
    *           <code>true</code> if it's the agent startup procedure, <code>false</code> if it's the session 
    *           startup procedure.
    * @param    startupProc
    *           The startup procedure.
    * @param    startupProcParam
    *           The startup procedure parameters.
    */
   private void runStartupProc(boolean isAgentStartupProc, String startupProc, String startupProcParam)
   {
      try
      {
         ControlFlowOps.invokePersistentSetWithMode(startupProc,
                                                    new handle(),
                                                    "I",
                                                    new character(startupProcParam));
      }
      catch (Throwable t)
      {
         if (!UnstoppableExitException.isCausing(t))
         {
            LegacyLogManager logManager = LegacyLogOps.logMgr();
            logStartupShutdownCondition(startupProc, t, logManager, true);

            int error = -6;
            
            if (t instanceof ErrorConditionException && 
               ((ErrorConditionException) t).getProgressErrorCode() == 8024)
            {
               error = -1;
            }
            
            String startupProcType = isAgentStartupProc ? "agent" : "session";
            logManager.writeMessage("Error (" + error + ") running " + startupProcType + " startup procedure.",
                                    LoggingEntryType.ASPlumbing,
                                    null,
                                    LoggingSubsys.MSAS,
                                    LoggingLevel.ERROR);

            logManager.writeMessage("Unable to initialize session!",
                                    LoggingEntryType.ASPlumbing,
                                    null,
                                    LoggingSubsys.MSAS,
                                    LoggingLevel.ERROR);
            
            if (LOG.isLoggable(Level.WARNING))
            {
               LOG.log(Level.WARNING,
                       t,
                       "Running " + startupProcType + " startup procedure %s has failed for appserver %s:",
                       startupProc,
                       definition.getAppserverName());
            }

            TransactionManager.popScope(); // remove the "startup" scope

            throw t;
         }
      }
   }

   /**
    * Creates and saves a session listener, then returns a Future that gives the id of the session 
    * received by the listener, when the session got free. Bound connections request a listener for a
    * specific session, while unbound connections wait in line to receive a notification for the next free 
    * unbound session. 
    * 
    * @param    connectionId
    *           The client connection id.
    * @param    sessionId
    *           The agent session id.
    *           
    * @return   Future returning the session id.
    */
   private Future<Integer> registerSessionListener(String connectionId, Integer sessionId)
   {
      if (state == State.TERMINATING)
      {
         return NO_SESSION_FOUND_RESULT;
      }
      
      SessionListenerFuture sessionListenerFuture = new SessionListenerFuture(connectionId, sessionId);

      synchronized (sessionsLock)
      {
         if (sessionId == null)
         {
            unboundSessionListeners.add(sessionListenerFuture);
         }
         else
         {
            boundSessionListeners.add(sessionListenerFuture);
         }
      }
      return sessionListenerFuture;
   }

   /**
    * Creates a new FWD context for a multi-session agent session and returns the id.
    * 
    * @return   The id of the newly created session (MSA security context). {@link #INVALID_SESSION_ID} if 
    *           a RestrictedUseException was thrown during creation.
    */
   private int createSession()
   {
      try
      {
         int contextId = msaSecurityManager.createContext(definition.getAppserverName(), 
                                                          definition.getAccountIds()[0]);
         sessionCreateTime.put(contextId, new Date());
         return contextId;
      }
      catch (RestrictedUseException e)
      {
         LOG.severe("RestrictedUseException in createSession.");
      }
      return INVALID_SESSION_ID;
   }
   
   /**
    * Sets the FWD context to the thread.
    *
    * @return   <code>true</code> if the context has been set successfully, <code>false</code> otherwise.
    */
   private boolean switchSession(String appserverName, int sessionId)
   {
      try
      {
         return msaSecurityManager.setContext(appserverName, sessionId);
      }
      catch (RestrictedUseException e)
      {
         LOG.severe("RestrictedUseException in switchSession.");
      }
      return false;
   }

   /**
    * Moves an agent session from idle to reserved state. 
    * 
    * @param    connectionId
    *           The client connection id.
    * @param    sessionId
    *           The agent session id.
    */
   private void reserveSession(String connectionId, int sessionId)
   {
      synchronized (sessionsLock)
      {
         if (isBound(sessionId))
         {
            idleBoundSessions.remove(sessionId);
         }
         else
         {
            idleSessions.remove(sessionId);
         }
         connIdByReservedSessionId.put(sessionId, connectionId);
      }
   }

   /**
    * Moves an agent session from reserved to active state. 
    *
    * @param    connectionId
    *           The client connection id.
    * @param    sessionId
    *           The agent session id.
    */
   private void useSession(String connectionId, int sessionId)
   {
      synchronized (sessionsLock)
      {
         connIdByActiveSessionId.put(sessionId, connectionId);
         connIdByReservedSessionId.remove(sessionId);
      }
   }

   /**
    * See {@link #freeSession(int, boolean, boolean)}.
    *
    * @param    sessionId
    *           The agent session id.
    * @param    callSessionListeners
    *           Flag to indicate the session listeners need to be called. <code>false</code> during session
    *           termination.
    */
   private void freeSession(int sessionId, boolean callSessionListeners)
   {
      freeSession(sessionId, callSessionListeners, false);
   }

   /**
    * Sets the agent session state to free. Removes the session from the active and reserved session 
    * collections. 
    * <br>
    * If in the meantime the appserver has been terminated, the session is in the process of 
    * shutdown, or the FWD context has been deleted, then the operation stops. If the appserver is in the 
    * process of terminating {@link #terminationSessionListener} is called.
    * <br>
    * If the max MSA session age is configured and the session is not bound, then a check if performed to 
    * decide if the session needs to be shutdown instead of added back to the collection of free sessions.
    * <br>
    * Bound and unbound sessions are handled in a similar way, but they are recorded in their respective 
    * idle session collection and call their respective session listeners.
    * 
    * @param    sessionId
    *           The agent session id.
    * @param    callSessionListeners
    *           Flag to indicate the session listeners need to be called. <code>false</code> during session
    *           termination.
    * @param    isSessionShutdown
    *           Flag to indicate the session is to be shut down.
    */
   private void freeSession(int sessionId, boolean callSessionListeners, boolean isSessionShutdown)
   {
      boolean isBound;
      synchronized (sessionsLock)
      {
         connIdByActiveSessionId.remove(sessionId);
         connIdByReservedSessionId.remove(sessionId);

         if (state == State.TERMINATED)
         {
            return;
         }

         boolean isDeleted = !msaSecurityManager.hasContext(definition.getAppserverName(), sessionId);
         if (isDeleted)
         {
            return;
         }

         if (state == State.TERMINATING)
         {
            if (terminationSessionListener != null)
            {
               terminationSessionListener.onSessionFreed(sessionId);
            }
            return;
         }

         if (isSessionShutdown)
         {
            return;
         }

         isBound = isBound(sessionId);
      }

      if (!isBound && definition.getMaxMsaSessionAge() > 0)
      {
         Date sessionCreateTime = this.sessionCreateTime.get(sessionId);
         if (sessionCreateTime.getTime() + definition.getMaxMsaSessionAge() < new Date().getTime())
         {
            if (LOG.isLoggable(Level.FINE))
            {
               LOG.fine("Old MSA session with id " + sessionId + " is to be shutdown.");
            }
            shutdownSession(MultiSessionAgent.SINGLE_AGENT_ID, 
                            sessionId, 
                            agentAdminConnection,
                            true, 
                            false, 
                            false);
            return;
         }
      }

      synchronized (sessionsLock)
      {
         if (!isBound)
         {
            idleSessions.add(sessionId);
            if (callSessionListeners)
            {
               Iterator<SessionListenerFuture> iterator = unboundSessionListeners.iterator();
               while (iterator.hasNext())
               {
                  SessionListener sessionListener = iterator.next();
                  if (sessionListener.isCancelled())
                  {
                     iterator.remove();
                     continue;
                  }
                  iterator.remove();
                  sessionListener.onSessionFreed(sessionId);
               }
            }
            return;
         }
         
         idleBoundSessions.add(sessionId);

         if (callSessionListeners)
         {
            Iterator<SessionListenerFuture> iterator = boundSessionListeners.iterator();
            while (iterator.hasNext())
            {
               SessionListener sessionListener = iterator.next();
               Integer boundSession = sessionListener.getRequestedSessionId();
               if (sessionListener.isCancelled())
               {
                  iterator.remove();
                  continue;
               }
               if (boundSession == sessionId)
               {
                  iterator.remove();
                  sessionListener.onSessionFreed(sessionId);
                  break;
               }
            }
         }
      }
   }

   /**
    * Returns <code>true</code> if the connection is bound, <code>false</code> otherwise.
    *
    * @param    connectionId
    *           The client connection id.
    *           
    * @return   See above.
    */
   private boolean isBound(String connectionId)
   {
      return connIdByBoundSessionId.containsValue(connectionId);
   }

   /**
    * Returns <code>true</code> if the connection is bound, <code>false</code> otherwise.
    *
    * @param    sessionId
    *           The agent session id.
    *
    * @return   See above.
    */
   private boolean isBound(int sessionId)
   {
      return connIdByBoundSessionId.containsKey(sessionId);
   }

   /**
    * Submits a {@link MsaTask} for execution. Tasks are not accepted during agent termination.
    * 
    * @param    task
    *           The MSA task to be executed.
    * 
    * @return   Future completed when the task execution is completed.
    * 
    * @throws   Exception
    *           An exception thrown when the task can't be accepted.
    */
   private Future submitTask(MsaTask task)
   throws Exception
   {
      return submitTask(task, false);
   }

   /**
    * Submits a {@link MsaTask} for execution.
    *
    * @param    task
    *           The MSA task to be executed.
    * @param    acceptDuringTermination
    *           Flag to indicate the task should be accepted during agent termination.
    *
    * @return   Future completed when the task execution is completed.
    *
    * @throws   Exception
    *           An exception thrown when the task can't be accepted.
    */
   private Future submitTask(MsaTask task, boolean acceptDuringTermination)
   throws Exception
   {
      if (!acceptDuringTermination && (state == State.TERMINATING || state == State.TERMINATED))
      {
         // TODO: replace with the OE error
         throw new Exception("Can't accept new tasks. The appserver " + definition.getAppserverName()
                                + " is not running.");
      }
      if (Thread.currentThread().getName().startsWith(WORKER_THREAD_NAME_PREFIX))
      {
         // sync exec on the same thread
         task.run();
         return CompletableFuture.completedFuture(null);
      }
      return workerPoolExecutor.submit(task);
   }

   /**
    * Shuts down the next session provided with the iterator.
    * 
    * @param    agentId
    *           The agent id.
    * @param    sessionIdIterator
    *           Iterator for session ids. Used to determine the next session to be shut down.
    * @param    finalizeCurrentContext
    *           Flag to indicate if the current context needs to be finalized after the session shut down.
    * @param    finalizeAgent
    *           Flag to indicate if the current agent needs to be terminated after the session shut down 
    *           (in effect only when the last session in the agent is shut down).
    * @param    acceptDuringTermination
    *           Flag to indicate the task should be accepted during agent termination.
    */
   private void shutdownNextSession(int agentId, 
                                    Iterator<Integer> sessionIdIterator,
                                    boolean finalizeCurrentContext,
                                    boolean finalizeAgent,
                                    boolean acceptDuringTermination)
   {
      Integer sessionId = sessionIdIterator.next();
      sessionIdIterator.remove();
      shutdownSession(agentId,
                      sessionId,
                      agentAdminConnection,
                      finalizeCurrentContext,
                      finalizeAgent,
                      acceptDuringTermination);
   }

   /**
    * Clears all references to the session and runs the session shutdown task.
    *
    * @param    agentId
    *           The agent id.
    * @param    sessionId
    *           The session id to be shut down.
    */
   private void shutdownSession(int agentId, int sessionId)
   {
      synchronized (sessionsLock)
      {
         // should not be added to the idle sessions, so don't use freeSession()
         idleSessions.remove(sessionId);
         idleBoundSessions.remove(sessionId);
         connIdByBoundSessionId.remove(sessionId);
         connIdByReservedSessionId.remove(sessionId);
         connIdByActiveSessionId.remove(sessionId);
      }
      
      shutdownSession(agentId,
                      sessionId,
                      agentAdminConnection,
                      true,
                      true,
                      false);
   }

   /**
    * Runs the session shutdown task.
    *
    * @param    agentId
    *           The agent id.
    * @param    sessionId
    *           The session id to be shut down.
    * @param    connection
    *           The connection (client or system).
    * @param    finalizeCurrentContext
    *           Flag to indicate if the current context needs to be finalized after the session shut down.
    * @param    finalizeAgent
    *           Flag to indicate if the current agent needs to be terminated after the session shut down 
    *           (in effect only when the last session in the agent is shut down).
    * @param    acceptDuringTermination
    *           Flag to indicate the task should be accepted during agent termination.
    * 
    * @return   Future completed when the session shutdown task is done.
    */
   private Future<?> shutdownSession(int agentId, 
                                     int sessionId,
                                     MultiSessionClientConnection connection,
                                     boolean finalizeCurrentContext, 
                                     boolean finalizeAgent,
                                     boolean acceptDuringTermination)
   {
      // before the shutdown call the session id has been removed from idle, reserved and active lists
      boolean executeAgentShutdownProc =
         finalizeAgent && !hasAnySession() && definition.getAgentShutdownProc() != null;

      if (!MultiSessionAppserverSecurityManager.getInstance()
                                               .hasContext(connection.getAppserverName(), sessionId))
      {
         LOG.log(Level.FINE, "Shutting down the session failed:");
         return CompletableFuture.completedFuture(false);
      }
      
      MsaTask task = new MsaTask(definition.getAppserverName(),
                                 agentId,
                                 sessionId,
                                 connection,
                                 null,
                                 null,
                                 false,
                                 null,
                                 null,
                                 MultiSessionClientConnection.AGENT_SYS_CONN_ID.equals(connection.getId()))
      {
         @Override
         void coreRun()
         {
            if (definition.getSessionShutdownProc() != null || executeAgentShutdownProc)
            {
               boolean isNestedCall = "startup".equals(TransactionManager.getScopeLabel());
               if (!isNestedCall)
               {
                  // add the topmost scope to the TransactionManager
                  TransactionManager.pushScope("startup",
                                               TransactionManager.NO_TRANSACTION,
                                               true,
                                               true,
                                               false,
                                               false);
               }

               if (definition.getSessionShutdownProc() != null)
               {
                  try
                  {
                     String sessionShutdownProc = definition.getSessionShutdownProc();
                     
                     log("shutdown Procedure '" + sessionShutdownProc + "' START (14244)",
                         LoggingLevel.VERBOSE,
                         LoggingEntryType.ASPlumbing,
                         null);
                     
                     ControlFlowOps.invokeExternalProcedure(new character(sessionShutdownProc),
                                                            false,
                                                            null,
                                                            false,
                                                            null,
                                                            null);

                     log("shutdown Procedure END SUCCESS. (14265)",
                         LoggingLevel.VERBOSE,
                         LoggingEntryType.ASPlumbing,
                         null);
                  }
                  catch (Exception e)
                  {
                     logShutdownProcError(definition.getSessionShutdownProc(), e);

                     if (!executeAgentShutdownProc)
                     {
                        if (!isNestedCall)
                        {
                           // remove the topmost scope from the TransactionManager
                           TransactionManager.popScope();
                        }

                        if (finalizeCurrentContext)
                        {
                           finalizeCurrentContext(sessionId);
                        }
                        
                        //throw e;
                        return; // task is running async without waiting for result, so just stop execution
                     }
                  }
               }

               if (executeAgentShutdownProc)
               {
                  try
                  {
                     ControlFlowOps.invokeExternalProcedure(new character(definition.getAgentShutdownProc()),
                                                            false,
                                                            null,
                                                            false,
                                                            null,
                                                            null);
                  }
                  catch (Exception e)
                  {
                     if (!isNestedCall)
                     {
                        // remove the topmost scope from the TransactionManager
                        TransactionManager.popScope();
                     }

                     logShutdownProcError(definition.getAgentShutdownProc(), e);

                     if (finalizeCurrentContext)
                     {
                        finalizeCurrentContext(sessionId);
                     }
                     
                     //throw e;
                     return; // task is running async without waiting for result, so just stop execution
                  }
               }

               if (!isNestedCall)
               {
                  // remove the topmost scope from the TransactionManager
                  TransactionManager.popScope();
               }
            }
            
            if (finalizeCurrentContext)
            {
               finalizeCurrentContext(sessionId);
            }
         }

         private void logShutdownProcError(String procName, Exception e)
         {
            if (!UnstoppableExitException.isCausing(e))
            {
               LegacyLogManager logManager = LegacyLogOps.logMgr();
               logStartupShutdownCondition(procName, e, logManager, false);
            }
         }
      };

      try
      {
         return submitTask(task, acceptDuringTermination);
      }
      catch (Exception e)
      {
         LOG.log(Level.FINE, "Shutting down the session failed:", e);
      }
      
      return null;
   }

   /**
    * Creates a legacy log for the the exception thrown by running the startup / shutdown agent / session 
    * procedure (all four variants).
    * 
    * @param    procName
    *           The procedure name.
    * @param    throwable
    *           The throwable to be handled.
    * @param    logManager
    *           The instance of the LOG-MANAGER.
    * @param    isStartupProc
    *           <code>true</code> if it's a startup procedure (agent or session).
    */
   private void logStartupShutdownCondition(String procName,
                                            Throwable throwable,
                                            LegacyLogManager logManager,
                                            boolean isStartupProc)
   {
      BlockManager.Condition conditionType;
      int errorNumber;
      if (throwable instanceof StopConditionException)
      {
         conditionType = BlockManager.Condition.STOP;
         errorNumber = 8026;
      }
      else if (throwable instanceof QuitConditionException)
      {
         conditionType = BlockManager.Condition.QUIT;
         errorNumber = 8027;
      }
      else
      {
         conditionType = BlockManager.Condition.ERROR;
         errorNumber = 8025;
      }

      String msg = procName + 
         (isStartupProc ? " startup" : " shutdown") + 
         " procedure ended with an " + conditionType.name() +
         " condition. (" + errorNumber + ")";
      logManager.writeMessage(msg, 
                              LoggingEntryType.ASPlumbing, 
                              null, 
                              LoggingSubsys.EMPTY,
                              LoggingLevel.ERROR);
   }

   /**
    * Runnable handling the agent termination.
    */
   private class AgentTerminationTask
   implements Runnable
   {
      /** The name of the appserver the agent belongs to. */
      private final String appserverName;

      /** The agent id. */
      private final int agentId;
      
      /** Timeout in milliseconds before the task is cancelled. */
      private final int timeoutMs;

      /** The exit code. */
      private final MultiSessionAgent.ExitCode exitCode;

      /** Flag indicating if the JVM is shutting down.. */
      private final boolean isJvmShuttingDown;

      /**
       * Package-private constructor.
       * 
       * @param    agentId
       *           The agent id.
       * @param    timeoutMs
       *           Timeout in milliseconds before the task is cancelled.
       * @param    exitCode
       *           The exit code.
       * @param    isJvmShuttingDown
       *           Flag indicating if the JVM is shutting down.
       */
      AgentTerminationTask(int agentId, 
                           int timeoutMs, 
                           MultiSessionAgent.ExitCode exitCode, 
                           boolean isJvmShuttingDown)
      {
         this.appserverName = definition.getAppserverName();
         this.agentId = agentId;
         this.timeoutMs = timeoutMs;
         this.exitCode = exitCode;
         this.isJvmShuttingDown = isJvmShuttingDown;
      }

      /**
       * Checks if the argument object represents the same task, identified by the all its fields.
       *
       * @param    o
       *           The object to compare to.
       *
       * @return   <code>true</code> if it's the same instance or all the fields are the same.
       *           <code>false</code> otherwise.
       */
      @Override
      public boolean equals(Object o)
      {
         if (this == o)
         {
            return true;
         }
         if (o == null || getClass() != o.getClass())
         {
            return false;
         }
         AgentTerminationTask that = (AgentTerminationTask) o;
         return agentId == that.agentId &&
            timeoutMs == that.timeoutMs &&
            exitCode == that.exitCode &&
            isJvmShuttingDown == that.isJvmShuttingDown &&
            Objects.equals(appserverName, that.appserverName);
      }

      /**
       * Returns the hash of the fields uniquely identifying the object.
       *
       * @return   See above.
       */
      @Override
      public int hashCode()
      {
         return Objects.hash(appserverName, agentId, timeoutMs, exitCode, isJvmShuttingDown);
      }

      /**
       * Runs all agent termination and cleanup actions. If no tasks are currently running and no tasks are
       * queued, completes immediately the Future waiting for shutting down the sessions. Otherwise 
       * registers a listener that gets notified when a session is freed and shuts it down. Starts a timer 
       * to calculate the remaining time waiting for the sessions to get shutdown. If the task times out 
       * the thread pool executed get forcefully shutdown. Finally the contexts for the appserver are removed 
       * from the security manager and all references removed from the appserver manager.
       */
      @Override
      public void run()
      {
         if (workerPoolExecutor == null || state == State.TERMINATING || state == State.TERMINATED)
         {
            return;
         }

         outOfContextLogManager.writeMessage("Agent Shutting Down. Status: " + exitCode.getNumber(),
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_LISTENER,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);

         if (!isJvmShuttingDown)
         {
            OrderedShutdownHooks.deregisterShutdownHook(1, this);
         }

         boolean isForcefulTermination = timeoutMs == 0;
         
         ElapsedTimer timeoutTimer = new ElapsedTimer(timeoutMs);
         timeoutTimer.start();

         CompletableFuture<Boolean> shuttingDownSessionsFuture = new CompletableFuture<>();

         synchronized (sessionsLock)
         {
            if (taskQueue.size() == 0 && workerPoolExecutor.getActiveCount() == 0 && !hasAnySession())
            {
               shuttingDownSessionsFuture.complete(true);
            }
            else
            {
               if (isForcefulTermination)
               {
                  taskQueue.clear();
               }
               
               terminationSessionListener = new TerminationSessionListener()
               {
                  private AtomicBoolean isCancelled = new AtomicBoolean(false);

                  @Override
                  public void onSessionFreed(int sessionId)
                  {
                     if (!isCancelled.get())
                     {
                        shutdownSession(agentId,
                                        sessionId,
                                        agentAdminConnection,
                                        true,
                                        true,
                                        true);
                     }
                  }

                  @Override
                  public void onSessionShutdown(int sessionId)
                  {
                     if (connIdByActiveSessionId.size() == 0 && taskQueue.size() == 0)
                     {
                        shuttingDownSessionsFuture.complete(true);
                     }
                  }

                  @Override
                  public boolean isCancelled()
                  {
                     return isCancelled.get();
                  }

                  /**
                   * If not already completed, completes this CompletableFuture with a {@link CancellationException}.
                   *
                   * @param    mayInterruptIfRunning
                   *           Flag to interrupt to thread.
                   *
                   * @return   {@code true} if this task is now cancelled
                   */
                  @Override
                  public boolean cancel(boolean mayInterruptIfRunning)
                  {
                     isCancelled.set(true);
                     return true;
                  }
               };
            }

            // should be after creating terminationSessionListener to not miss a session event
            state = State.TERMINATING;

            for (Iterator<Integer> it = idleSessions.iterator(); it.hasNext(); )
            {
               shutdownNextSession(agentId, it, true, true, true);
            }
            for (Iterator<Integer> it = idleBoundSessions.iterator(); it.hasNext(); )
            {
               shutdownNextSession(agentId, it, true, true, true);
            }
            for (Iterator<Integer> it = connIdByBoundSessionId.keySet().iterator(); it.hasNext(); )
            {
               shutdownNextSession(agentId, it, true, true, true);
            }
         }

         for (SessionListenerFuture unboundSessionListener : unboundSessionListeners)
         {
            unboundSessionListener.complete(INVALID_SESSION_ID);
         }
         unboundSessionListeners.clear();

         for (SessionListenerFuture boundSessionListener : boundSessionListeners)
         {
            boundSessionListener.complete(INVALID_SESSION_ID);
         }
         boundSessionListeners.clear();
         
         try
         {
            long left = timeoutTimer.getLeft();
            if (LOG.isLoggable(Level.FINE))
            {
               LOG.fine("Waiting for shutting down all sessions for the next " + left +
                           " milliseconds.");
            }
            shuttingDownSessionsFuture.get(left, TimeUnit.MILLISECONDS);

            synchronized (sessionsLock)
            {
               if (terminationSessionListener != null)
               {
                  terminationSessionListener.cancel(true);
                  terminationSessionListener = null;
               }
            }
         }
         catch (Exception ignored)
         {
         }
         
         if (!workerPoolExecutor.isTerminated())
         {
            long left = timeoutTimer.getLeft();
            if (left == 0)
            {
               workerPoolExecutor.shutdownNow();
            }
            else
            {
               workerPoolExecutor.shutdown();
               try
               {
                  if (LOG.isLoggable(Level.FINE))
                  {
                     LOG.fine("Waiting for shutting down the worker executor for the next " + left +
                                 " milliseconds.");
                  }
                  workerPoolExecutor.awaitTermination(left, TimeUnit.MILLISECONDS);
               }
               catch (InterruptedException ignored)
               {
               }
            }
         }
         workerPoolExecutor = null;

         try
         {
            // move outside when multiple agents implemented
            msaSecurityManager.removeApp(definition.getAppserverName());
            MultiSessionAppserverManager.getInstance().clearApp(definition.getAppserverName());
            sessionCreateTime.clear();
         }
         catch (Throwable t)
         {
            LOG.warning("", t);
         }

         state = State.TERMINATED;

         outOfContextLogManager.writeMessage("Agent Shutdown Complete.",
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_LISTENER,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);
      }
   }

   /**
    * Returns <code>true</code> if there are any existing agent sessions, <code>false</code> otherwise.
    * 
    * @return   See above.
    */
   private boolean hasAnySession()
   {
      synchronized (sessionsLock)
      {
         return idleSessions.size() > 0 || idleBoundSessions.size() > 0 || 
            connIdByReservedSessionId.size() > 0 || connIdByActiveSessionId.size() > 0 || 
            connIdByBoundSessionId.size() > 0;
      }
   }

   /**
    * Resets the agent session / FWD context.
    * 
    * @param    agentId
    *           The agent id.
    * @param    sessionId
    *           The agent session id.
    * @param    connection
    *           The client connection.
    */
   private void resetSession(Integer agentId, Integer sessionId, MultiSessionClientConnection connection)
   {
      /* the MSA session is reset to its initial state, which includes deletion of persistent procedures and 
      static MSA objects, the disconnection of databases (or re-connection if the databases were connected 
      at startup), and the clean-up of all global data, such as shared variables */
      try
      {
         shutdownSession(agentId, sessionId, connection, false, false, false);
      }
      catch (Exception e)
      {
         if (LOG.isLoggable(Level.FINE))
         {
            LOG.log(Level.FINE, "Session shutdown proc throws during session reset.", e);
         }
      }

      // before cleaning up, cleanup all still-alive dynamic frames
      // TODO(with UI support): LogicalTerminal.cleanupDynamicFrames();

      // pop the global SESSION_GLOBAL_SCOPE scope, which will be pushed again in the 'startupSession' method.  
      TransactionManager.popScope();
      
      try
      {
         msaSecurityManager.resetContext(definition.getAppserverName(), sessionId);
      }
      catch (RestrictedUseException e)
      {
         LOG.severe("RestrictedUseException in resetSession.");
      }
      
      try
      {
         startupSession(agentId, sessionId, connection, false, true);
      }
      catch (Exception e)
      {
         shutdownSession(agentId, sessionId, connection, true, true, false);
      }
   }

   /**
    * Finalizes the current session context by removing references and deleting the FWD context. 
    * 
    * @param    sessionId
    *           The agent session id.
    */
   private void finalizeCurrentContext(int sessionId)
   {
      synchronized (sessionsLock)
      {
         // should not signal the session listeners, so don't use freeSession()
         connIdByActiveSessionId.remove(sessionId);
         sessionCreateTime.remove(sessionId);
      }

      try
      {
         msaSecurityManager.deleteCurrentContext(definition.getAppserverName(), sessionId);
      }
      catch (RestrictedUseException e)
      {
         LOG.warning("", e);
      }

      if (state == State.TERMINATING && terminationSessionListener != null)
      {
         terminationSessionListener.onSessionShutdown(sessionId);
      }
   }

   /**
    * The thread to be used by the appserver thread pool executor.
    */
   private class MsaThread
   extends Thread
   {
      /** The incremental number of the thread. */
      private final int threadNumber;
      
      /** LOG-MANAGER instance used for logging appserver logs out of appserver context. */
      private final LegacyLogManager outOfContextLogManager;

      /**
       * Package-private constructor.
       * 
       * @param    target
       *           The thread runnable.
       * @param    name
       *           The thread name.
       * @param    outOfContextLogManager
       *           LOG-MANAGER instance used for logging appserver logs out of appserver context.
       */
      MsaThread(Runnable target, String name, LegacyLogManager outOfContextLogManager)
      {
         super(target, name);
         this.threadNumber = workerNumberCounter.getAndIncrement();
         this.outOfContextLogManager = outOfContextLogManager;
      }

      /**
       * Sets the FWD context on the thread, runs the thread and creates logs for the state of the thread.
       */
      @Override
      public void run()
      {
         try
         {
            MultiSessionAppserverSecurityManager.setInitialContext();
         }
         catch (RestrictedUseException e)
         {
            LOG.warning("", e);
         }

         if (LOG.isLoggable(Level.FINEST))
         {
            LOG.finest("Starting thread " + getName());
         }

         outOfContextLogManager.writeMessage("Spawning New Worker Thread. Number: " + threadNumber,
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_LISTENER,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);
         
         super.run();

         outOfContextLogManager.writeMessage("Worker Thread exiting. Number: " + threadNumber,
                                             LoggingEntryType.ASPlumbing,
                                             LoggingExecEnv.AS_AUX,
                                             LoggingSubsys.MSAS,
                                             LoggingLevel.ERROR);

         if (LOG.isLoggable(Level.FINEST))
         {
            LOG.finest("Stopping thread " + getName());
         }
      }
   }

   /**
    * The thread factory for the appserver thread pool executor.
    */
   private class MsaThreadFactory
   implements ThreadFactory
   {
      /** The counter for the thread ids. */
      private final AtomicInteger count = new AtomicInteger(1);

      /** LOG-MANAGER instance used for logging appserver logs out of appserver context. */
      private final LegacyLogManager outOfContextLogManager;

      /**
       * Package-private constructor.
       * 
       * @param    outOfContextLogManager
       *           LOG-MANAGER instance used for logging appserver logs out of appserver context.
       */
      MsaThreadFactory(LegacyLogManager outOfContextLogManager)
      {
         this.outOfContextLogManager = outOfContextLogManager;
      }

      /**
       * Creates a new thread with a MSA specific name.
       * 
       * @param    runnable
       *           The runnable for the new thread.
       * 
       * @return   The newly created thread.
       */
      @Override
      public Thread newThread(@Nonnull Runnable runnable)
      {
         return new MsaThread(runnable,
                              WORKER_THREAD_NAME_PREFIX + count.getAndIncrement() + 
                                   " for app " + definition.getAppserverName(),
                              outOfContextLogManager);
      }
   }

   /**
    * An abstract class encapsulating the core MSA task functionality.
    */
   abstract class MsaTask
   implements Runnable
   {
      /** The unique task id. */
      private final int id;

      /** Flag indicating if the request is running on a bound session. */
      private final boolean bound;

      /** The connection id. */
      private final MultiSessionClientConnection connection;

      /** The agent id. */
      private final int agentId;

      /** The session id. */
      private final int sessionId;

      /** The appserver name. */
      private final String appserverName;

      /** The result for the request. */
      private final AppServerInvocationResult result;

      /** The name of the running program. */
      private final String runningProgram;

      /** Thread-safe supplier indicating if this MSA task is running. */
      private final AtomicBoolean isRunning;

      /** Thread-safe supplier indicating if the wrapper task is cancelled. */
      private final AtomicBoolean isTaskCancelled;
      
      /** Flag indicating if it's an initialization or finalization system task. */
      private final boolean isInitialOrFinalSetup;

      /** Stacktrace of the original thread of the task. Used for debugging. */
      private final List<StackTraceElement[]> debugStackTrace;
      
      /** Instance of the multi-session appserver manager. */
      private final MultiSessionAppserverManager msaManager = MultiSessionAppserverManager.getInstance();
      
      /** Weak reference to the executor thread this task is running on. */
      private volatile WeakReference<Thread> executorThreadRef;

      /**
       * Package-private constructor.
       * 
       * @param    appserverName
       *           The appserver name.
       * @param    agentId
       *           The agent id.
       * @param    sessionId
       *           The session id.
       * @param    connection
       *           The client connection.
       * @param    result
       *           The result for the request.
       * @param    runningProgram
       *           The name of the running program.
       * @param    bound
       *           Flag indicating if the request is running on a bound session.
       * @param    isRunning
       *           Thread-safe supplier indicating if this MSA task is running.
       * @param    isTaskCancelled
       *           Thread-safe supplier indicating if the wrapper task is cancelled.
       * @param    isInitialOrFinalSetup
       *           Flag indicating if it's an initialization or finalization system task.
       */
      MsaTask(String appserverName,
              int agentId,
              int sessionId,
              MultiSessionClientConnection connection,
              AppServerInvocationResult result,
              String runningProgram,
              boolean bound,
              AtomicBoolean isRunning,
              AtomicBoolean isTaskCancelled,
              boolean isInitialOrFinalSetup)
      {
         this.id = msaTaskIdGenerator.getAndIncrement();
         this.appserverName = appserverName;
         this.agentId = agentId;
         this.sessionId = sessionId;
         this.connection = connection;
         this.bound = bound;
         this.result = result;
         this.runningProgram = runningProgram;
         this.isRunning = isRunning;
         this.isTaskCancelled = isTaskCancelled;
         this.isInitialOrFinalSetup = isInitialOrFinalSetup;
         
         if (!definition.isDebugMsa())
         {
            this.debugStackTrace = null;
         }
         else
         {
            this.debugStackTrace = new ArrayList<>();
            List<StackTraceElement[]> parentTaskDebugStackTrace = msaManager.getDebugStackTrace();
            if (parentTaskDebugStackTrace != null)
            {
               this.debugStackTrace.addAll(parentTaskDebugStackTrace);
            }
            this.debugStackTrace.add(Thread.currentThread().getStackTrace());
         }
      }

      /**
       * Runs the implementation specific core behavior.
       */
      abstract void coreRun();

      /**
       * Prepares the task and runs the implementation specific actions via {@link #coreRun()}. If the 
       * wrapper task has been cancelled before the MsaTask is started by the executor, then it doesn't run.
       * Otherwise a weak reference to the executor thread is created, the {@link #isRunning} flag is set 
       * to true, the FWD context is set on the thread, the agent session state is set to active, context 
       * local vars are set in the appserver manager to allow access to the current session state from the 
       * executing code, the currently running task and program are set to the connection and the core task
       * is run. Generic MSA error handling, logging and cleanup are performed. In case this task is a 
       * nested task (started by another MsaTask) the original context is stored before coreRun is called,
       * and on completion restored to allow the original task completing in the right context.
       */
      @Override
      public final void run()
      {
         if (isTaskCancelled != null && isTaskCancelled.get())
         {
            return;
         }
         
         executorThreadRef = new WeakReference<>(Thread.currentThread());

         if (isRunning != null)
         {
            isRunning.set(true);
         }

         MultiSessionClientConnection originalConnection = msaManager.getConnection();
         Integer originalAgentId = msaManager.getAgentId();
         Integer originalSessionId = msaManager.getSessionId();
         boolean originalIsInitialSetup = msaManager.isInitialSetup();
         List<StackTraceElement[]> originalDebugStackTrace = msaManager.getDebugStackTrace();

         if (!switchSession(appserverName, sessionId))
         {
            executorThreadRef = null;
            outOfContextLogManager.writeMessage("Session AS-" + sessionId + 
                                                   " could not be found. Cannot process request.",
                                                LoggingEntryType.ASPlumbing,
                                                LoggingExecEnv.AS_AUX,
                                                LoggingSubsys.MSAS,
                                                LoggingLevel.ERROR);
            // 1 AS-Aux-0 MSAS Worker Thread exiting. Number: 4, Status: -14
            throw new ErrorConditionException("Error handling request! Status=-1003");
         }

         useSession(connection.getId(), sessionId);
         
         msaManager.setMsaContext(connection, agentId, sessionId, isInitialOrFinalSetup, debugStackTrace);
         
         connection.setCurrentlyRunningTask(this);
         if (runningProgram != null)
         {
            connection.setCurrentlyRunningProgram(runningProgram);
         }
         
         if (LOG.isLoggable(Level.FINE))
         {
            LOG.fine("MsaTask #" + id + " starting on thread `" + Thread.currentThread().getName() + 
                        "` for agentId: " + agentId + 
                        ", sessionId: " + sessionId + 
                        ", connectionId: " + connection.getId());
         }
         
         boolean isQuit = false;
         try 
         {
            coreRun();
            if (result != null && result.getError() instanceof QuitConditionException)
            {
               isQuit = true;
               msaManager.resetMsaContext();
               resetSession(agentId, this.sessionId, connection);
            }
         }
         catch (QuitConditionException e)
         {
            isQuit = true;
            msaManager.resetMsaContext();
            resetSession(agentId, this.sessionId, connection);
            throw e;
         }
         finally
         {
            if (!isQuit)
            {
               msaManager.resetMsaContext();
            }
            executorThreadRef = null;

            connection.setCurrentlyRunningTask(null);
            connection.setCurrentlyRunningProgram(null);

            if (LOG.isLoggable(Level.FINE))
            {
               LOG.fine("MsaTask #" + id + " ends for agentId: " + agentId + 
                           ", sessionId: " + this.sessionId + 
                           ", connectionId: " + connection.getId());
            }
            
            if (originalSessionId != null)
            {
               switchSession(appserverName, originalSessionId);
               msaManager.setMsaContext(originalConnection,
                                        originalAgentId,
                                        originalSessionId,
                                        originalIsInitialSetup,
                                        originalDebugStackTrace);
            }
         }
      }

      /**
       * Returns the string representation.
       *
       * @return   See above.
       */
      @Override
      public String toString()
      {
         return "MsaTask{" +
            "bound=" + bound +
            ", sessionId=" + sessionId +
            ", connectionId='" + connection.getId() + '\'' +
            '}';
      }

      /**
       * Returns the task id.
       *
       * @return   See above.
       */
      public int getId()
      {
         return id;
      }

      /**
       * Returns the thread this task is currently running on. Can be <code>null</code> if the task hasn't 
       * been started or is already completed.
       * 
       * @return   See above.
       */
      Thread getRunningThread()
      {
         if (executorThreadRef == null)
         {
            return null;
         }
         return executorThreadRef.get();
      }
      
      /**
       * Sends STOP by cancelling the task and interrupting the running thread.
       */
      void sendStop()
      {
         Thread thread = executorThreadRef.get();
         if (thread != null)
         {
            thread.interrupt();
         }
      }
   }

   /**
    * Interface of a listener listening for freed sessions during agent termination.
    */
   private interface TerminationSessionListener
   {
      /**
       * Called when a session is freed. 
       * 
       * @param    sessionId
       *           The session id.
       */
      void onSessionFreed(int sessionId);

      /**
       * Called when a session has been shutdown.
       * 
       * @param    sessionId
       *           The session id.
       */
      void onSessionShutdown(int sessionId);

      /**
       * If not already completed, completes this CompletableFuture with a {@link CancellationException}.
       *
       * @param    mayInterruptIfRunning
       *           Flag to interrupt to thread.
       *
       * @return   {@code true} if this listener is now cancelled
       */
      boolean cancel(boolean mayInterruptIfRunning);

      /**
       * Check if the listener is cancelled.
       *
       * @return   {@code true} if this listener is cancelled
       */
      boolean isCancelled();
   }

   /**
    * Interface of a listener listening for freed sessions.
    */
   private interface SessionListener
   {
      /**
       * Called when a session is freed. 
       *
       * @param    sessionId
       *           The session id.
       */
      void onSessionFreed(int sessionId);

      /**
       * Returns the id of session requested to be reserved. <code>null</code> when the connection is not 
       * bound.
       * 
       * @return   See above.
       */
      Integer getRequestedSessionId();

      /**
       * Check if the listener is cancelled.
       *
       * @return   {@code true} if this listener is cancelled
       */
      boolean isCancelled();
   }

   /**
    * Future completing when the session it listens for is free. Returns the session id on completion.
    */
   private class SessionListenerFuture
   extends CompletableFuture<Integer>
   implements SessionListener
   {
      /** A unique id for the instance. */
      private final String uniqueId = UUID.randomUUID().toString();

      /** The id of the connection that wants to reserve the session. */
      private final String connectionId;

      /** The id of session requested to be reserved. <code>null</code> when the connection is not bound. */
      private final Integer requestedSessionId;

      /**
       * Package-private constructor.
       * 
       * @param    connectionId
       *           The id of the connection that wants to reserve the session.
       * @param    sessionId
       *           The id of session requested to be reserved. <code>null</code> when the connection is not
       *           bound.
       */
      SessionListenerFuture(String connectionId, Integer sessionId)
      {
         this.connectionId = connectionId;
         this.requestedSessionId = sessionId;
      }

      /**
       * If not already completed, completes this CompletableFuture with a {@link CancellationException}.
       *
       * @param    mayInterruptIfRunning
       *           Flag to interrupt to thread.
       *
       * @return   {@code true} if this task is now cancelled
       */
      @Override
      public boolean cancel(boolean mayInterruptIfRunning)
      {
         synchronized (sessionsLock)
         {
            complete(INVALID_SESSION_ID);

            synchronized (sessionsLock)
            {
               unboundSessionListeners.remove(this);
               boundSessionListeners.remove(this);
            }
            
            return super.cancel(mayInterruptIfRunning);
         }
      }

      /**
       * Called when a session that meets requirements gets free. This completes the Future.
       * 
       * @param    sessionId
       *           The id of session that is freed.
       */
      @Override
      public void onSessionFreed(int sessionId)
      {
         reserveSession(connectionId, sessionId);
         complete(sessionId);
      }

      /**
       *
       * @return   The id of the session that is requested.
       */
      @Override
      public Integer getRequestedSessionId()
      {
         return requestedSessionId;
      }

      /**
       * Checks if the argument object represents the same listener, identified by the connectionId and 
       * sessionId.
       *
       * @param    o
       *           The object to compare to.
       *
       * @return   <code>true</code> if it's the same instance or the connectionId and sessionId are the same.
       *           <code>false</code> otherwise.
       */
      @Override
      public boolean equals(Object o)
      {
         if (this == o)
         {
            return true;
         }
         if (o == null || getClass() != o.getClass())
         {
            return false;
         }
         SessionListenerFuture that = (SessionListenerFuture) o;
         return Objects.equals(uniqueId, that.uniqueId);
      }

      /**
       * Returns the hash of the fields uniquely identifying the object.
       *
       * @return   See above.
       */
      @Override
      public int hashCode()
      {
         return Objects.hash(uniqueId);
      }
   }
}