MultiSessionAppserverManager.java

/*
** Module   : MultiSessionAppserverManager.java
** Abstract : Manager for appserver with multi-session agents.
**
** Copyright (c) 2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------Description----------------------------------
** 001 GBB 20250122 Created initial version.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
**
**   0. Attribution Requirement.
**
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
**
**   1. No License To Use Trademarks.
**
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
**
**   2. No Misrepresentation of Affiliation.
**
**     You may not represent yourself as Golden Code Development Corporation or FWD.
**
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
**
**   3. No Misrepresentation of Source or Origin.
**
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util.appserver;

import com.goldencode.p2j.*;
import com.goldencode.p2j.admin.*;
import com.goldencode.p2j.main.*;
import com.goldencode.p2j.oo.web.*;
import com.goldencode.p2j.rest.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.util.*;
import com.goldencode.p2j.util.logging.*;

import javax.servlet.http.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.function.*;
import java.util.logging.*;
import java.util.stream.*;

/** Manager for appserver with multi-session agents. */
public class MultiSessionAppserverManager
{
   /** Initialization on demand idiom singleton holder */
   private static class SingletonHolder
   {
      /** Singleton instance */
      private static final MultiSessionAppserverManager INSTANCE = new MultiSessionAppserverManager();
   }
   
   /** Context-local global data area for MSA session state */
   private static final ContextLocal<WorkArea> local = new ContextLocal<WorkArea>()
   {
      /**
       * Get the weight of this context-local var.
       *
       * @return   The weight of this context-local var.
       */
      @Override
      public WeightFactor getWeight()
      {
         return WeightFactor.LAST;
      }
      
      /**
       * Initializes the work area, the first time it is requested within a
       * new context.
       *
       * @return   The newly instantiated work area.
       */
      @Override
      protected synchronized WorkArea initialValue()
      {
         return new WorkArea();
      }
   };

   /** Decimal representation of digit characters is in range [48:57], for uppercase latin letters [65:90]. */
   private static final int[] CONNECTION_ID_CHARS = IntStream.concat(IntStream.rangeClosed(48, 57),
                                                                     IntStream.rangeClosed(65, 90))
                                                             .toArray();

   /** The number of random characters in the connection id. */
   private static final int CONNECTION_ID_RANDOM_CHAR_COUNT = 44;

   /** Thread local flag to indicate if it's a multi-session agent context */
   private static final ThreadLocal<Boolean> isMsaContext = ThreadLocal.withInitial(() -> false);
   
   /** Logger. */
   private static final CentralLogger LOG = CentralLogger.get(MultiSessionAppserverManager.class);

   /** No connections error msg. */
   private static final String ERR_NO_CONNECTIONS = "No connections available to process request.";

   /** Value to disable the client driven session execution timeout. */
   private static final int DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT = -1;
   
   /** Pairs of instantiated appserver name : instance. */
   private final Map<String, MultiSessionAppserver> startedAppservers = new ConcurrentHashMap<>();

   /** Pairs of connection id : connection object. */
   private final Map<String, MultiSessionClientConnection> connections = new ConcurrentHashMap<>();

   /** Instance of the scheduled executor where timeout jobs are running for the requests. */
   private final ScheduledExecutorService TIMEOUT_EXECUTOR = 
      Executors.newScheduledThreadPool(1, new ThreadFactory()
   {
      /** Thread-safe counter for the thread ids used to run the timeout jobs. */
      private final AtomicInteger threadIdCounter = new AtomicInteger(0);
      
      /**
       * Constructs a new {@code Thread}.
       *
       * @param    runnable 
       *           A runnable to be executed by new thread instance.
       *                     
       * @return   Constructed thread.
       */
      @Override
      public Thread newThread(Runnable runnable)
      {
         Thread newThread = new Thread(runnable,
                                       "MSA Timeout Executor thread " + threadIdCounter.getAndIncrement());
         newThread.setDaemon(true);
         return newThread;
      }
   });
   
   /** 
    * Consumer anonymous instance accepting connection id and program handle and associating the persistent 
    * resource with the connection.
    */
   private final BiConsumer<String, handle> ADD_PROC_FUNC = (connectionId, extProg) ->
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection != null)
      {
         connection.addPersistentProc(extProg);
      }
   };

   /**
    * Consumer anonymous instance accepting connection id and program res id and removing the associating of 
    * the persistent resource from the connection.
    */
   private final BiConsumer<String, String> REMOVE_PROC_FUNC = (connectionId, procId) ->
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection != null)
      {
         connection.deletePersistentProc(procId);
         
         // TODO: 
         /*if (connection.isBound())
         {
            msaAppserver.unbind(connection, sessionId, true);

            CoreAppserver.invokeDeactivate(msaDef.getSessionDeactivateProc(),
                                           isConnectionBound(),
                                           connectionId,
                                           SET_RUNNING_PROC_FUNC);
         }*/
      }
   };
   
   /**
    * Consumer anonymous instance accepting connection id and disconnecting the client connection from the 
    * appserver.
    */
   private final Consumer<String> DISCONNECT_FUNC = (connId) -> AppServerManager.disconnect(null, connId);

   /**
    * Consumer anonymous instance accepting connection id and session id and binding the MSA session to the
    * connection. 
    */
   private final BiConsumer<String, Integer> BIND_PERSISTENT_FUNC = (connectionId, sessionId) ->
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection == null || connection.isDisconnecting())
      {
         return;
      }
      MultiSessionAppserver msaAppserver = startedAppservers.get(connection.getAppserverName());
      if (msaAppserver == null)
      {
         return;
      }
      msaAppserver.bind(connection, sessionId, true);
   };

   /**
    * Consumer anonymous instance accepting connection id and session id and unbinding the MSA session from
    * the connection. 
    */
   private final BiConsumer<String, Integer> UNBIND_PERSISTENT_FUNC = (connectionId, sessionId) ->
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection == null || connection.isDisconnecting())
      {
         return;
      }
      MultiSessionAppserver msaAppserver = startedAppservers.get(connection.getAppserverName());
      if (msaAppserver == null)
      {
         return;
      }
      msaAppserver.unbind(connection, sessionId, true);
   };

   /**
    * Consumer anonymous instance accepting connection id and the name of the currently running program and 
    * creating association between them. 
    */
   private final BiConsumer<String, String> SET_RUNNING_PROC_FUNC = (connectionId, procName) ->
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection != null)
      {
         connection.setCurrentlyRunningProgram(procName);
      }
   };

   /** The max number of timed out client connections to be kept in the cache. */
   private static final int MAX_CACHED_TIMED_OUT_CONNECTIONS = 100;
   
   /** Collection of ids of times out client connections */
   private final Map<String, MultiSessionClientConnection> timedOutConnections = 
      new LinkedHashMap<String, MultiSessionClientConnection>(MAX_CACHED_TIMED_OUT_CONNECTIONS * 4/3, 
                                                              0.75f, 
                                                              false)
      {
         @Override
         protected boolean removeEldestEntry(Map.Entry<String, MultiSessionClientConnection> eldest)
         {
            return size() > MAX_CACHED_TIMED_OUT_CONNECTIONS;
         }
      };
   
   /**
    * Consumer anonymous instance accepting a collection of client connections that need to be disconnected
    * due to being timed out.
    */
   private final Consumer<Collection<MultiSessionClientConnection>> REMOVE_TIMEDOUT_CONN_FUNC = clientConnections ->
   {
      boolean isInfoLevelLoggable = LOG.isLoggable(Level.INFO);
      for (MultiSessionClientConnection connection : clientConnections)
      {
         String id = connection.getId();
         if (isInfoLevelLoggable)
         {
            LOG.log(Level.INFO, "Client connection with id %s timed out and will be disconnected.", id);
         }

         synchronized (timedOutConnections)
         {
            timedOutConnections.put(id, connection);
         }

         disconnect(id, true);
      }
   };
   
   /**
    * Static method for access to the singleton instance of the manager 
    *
    * @return  The singleton instance of the manager 
    */
   public static MultiSessionAppserverManager getInstance()
   {
      return SingletonHolder.INSTANCE;
   }

   /**
    * Returns the number of MSA agents running for the specified appserver.
    *
    * @param   appserverName
    *          The appserver name.
    * 
    * @return  See above.
    */
   public int getNumberOfAgents(String appserverName)
   {
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      return msaAppserver == null ? 0 : msaAppserver.getNumberOfAgents();
   }

   /**
    * Returns the number of MSA sessions running for the specified appserver.
    *
    * @param   appserverName
    *          The appserver name.
    *
    * @return  See above.
    */
   public int getNumberOfSessions(String appserverName)
   {
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      return msaAppserver == null ? 0 : msaAppserver.getNumberOfSessions();
   }
   
   /**
    * Returns <code>true</code> if the current security context is for MSA session.
    *
    * @return  See above.
    */
   public boolean isMsaContext()
   {
      return isMsaContext.get();
   }

   /**
    * Returns <code>true</code> if the specified connection id is for MSA.
    *
    * @param   connectionId
    *          The connection id.
    *          
    * @return  See above.
    */
   public boolean isMsaConnection(String connectionId)
   {
      return connectionId.indexOf(".") == CONNECTION_ID_RANDOM_CHAR_COUNT;
   }

   /**
    * Returns the MSA client connection with the specified id.
    *
    * @param   connectionId
    *          The connection id.
    *
    * @return  See above.
    */
   public MultiSessionClientConnection getConnection(String connectionId)
   {
      return connections.get(connectionId);
   }

   /**
    * Returns <code>true</code> if the current client connection is running in Session-Free.
    *
    * @return  See above.
    */
   public boolean isSessionFree()
   {
      return local.get().connection.isSessionFree();
   }

   /**
    * Returns the appserver name executing the currently running request.
    *
    * @return  See above.
    */
   public String getAppserverName()
   {
      MultiSessionClientConnection connection = local.get().connection;
      return connection == null ? null : connection.getAppserverName();
   }

   /**
    * Returns the connection id for the currently running request.
    *
    * @return  See above.
    */
   public String getConnectionId()
   {
      MultiSessionClientConnection connection = local.get().connection;
      return connection == null ? null : connection.getId();
   }

   /**
    * Returns <code>true</code> if the current client connection is bound to an MSA session.
    *
    * @return  See above.
    */
   public boolean isConnectionBound()
   {
      return local.get().connection.isBound();
   }

   /**
    * Returns the MSA session id used for the currently running request.
    *
    * @return  See above.
    */
   public Integer getSessionId()
   {
      return local.get().sessionId;
   }
   
   /**
    * Returns the agent id used for the currently running request.
    *
    * @return  See above.
    */
   public Integer getAgentId()
   {
      return local.get().agentId;
   }

   /**
    * Returns the client connection of the currently running request.
    *
    * @return  See above.
    */
   public MultiSessionClientConnection getConnection()
   {
      return local.get().connection;
   }

   /**
    * Returns <code>true</code> if the currently running task is initial setup for the session.
    *
    * @return  See above.
    */
   public boolean isInitialSetup()
   {
      return local.get().isStartupProc;
   }

   /**
    * Returns the stacktrace of the original thread of the MsaTask running on the context. Used for debugging.
    *
    * @return  See above.
    */
   public List<StackTraceElement[]> getDebugStackTrace()
   {
      return local.get().debugStackTrace;
   }

   /**
    * Returns the persistent procedure name resolved by its resource id on the connection specified by the 
    * connection id.
    *
    * @param   connectionId
    *          The connection id.
    * @param   remoteProcResId
    *          The remote procedure resource id.
    * @param   agentId
    *          The agent id.
    *          
    * @return  See above.
    */
   public String getProcedureName(String connectionId, String remoteProcResId, int agentId)
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      return connection == null ? null : connection.getPersistentProcName(remoteProcResId);
   }
   
   /**
    * Get the SERVER-CONNECTION-CONTEXT attribute of this session.
    *
    * @return   See above.
    */
   public static String getServerConnectionContext()
   {
      return local.get().connection.getServerConnectionContext();
   }

   /**
    * Set the SERVER-CONNECTION-CONTEXT attribute of this session.
    *
    * @param    value
    *           The attribute's value.
    */
   public static void setServerConnectionContext(String value)
   {
      local.get().connection.setServerConnectionContext(value);
   }

   /**
    * Method that specifies the remote procedures the client can execute from the AppServer 
    * session.
    *
    * @param    procList
    *           The list of procedure  and name-patterns separated by comma.
    *
    * @return   <code>true</code> if the method succeeded.
    */
   public logical export(character procList)
   {
      MultiSessionClientConnection currentConnection = local.get().connection;
      if (currentConnection == null)
      {
         return new logical(false);
      }
      MultiSessionAppserver currentAppserver = startedAppservers.get(currentConnection.getAppserverName());
      if (currentAppserver == null)
      {
         return new logical(false);
      }
      return currentAppserver.setExports(procList);
   }

   /**
    * Collects and returns the info for the appserver agent. 
    *
    * @param    appserverName
    *           The appserver name to trim.
    *
    * @return   The info for the MSA agent.
    */
   public MultiSessionAgentInfo getAgentInfo(String appserverName)
   {
      if (!startedAppservers.containsKey(appserverName))
      {
         return null;
      }
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      return new MultiSessionAgentInfo(msaAppserver.getStartupTime(),
                                       MultiSessionAgent.SINGLE_AGENT_ID,
                                       msaAppserver.getSessionDefs());
   }

   /**
    * Check if current running appserver knows the given procedure.
    *
    * @param    referent
    *           A persistent procedure referent.
    *
    * @return   <code>true</code> if the persistent procedure is known to the current running appserver.
    */
   public boolean hasProcedure(Object referent)
   {
      MultiSessionClientConnection currentConnection = local.get().connection;
      if (currentConnection == null)
      {
         return false;
      }
      return currentConnection.hasPersistentProc(referent);
   }

   /**
    * Returns <code>true</code> if the appserver specified by the name starts successfully, or is already 
    * running.
    *
    * @param   appserverName
    *          The appserver name.
    *
    * @return  See above.
    */
   public boolean start(String appserverName)
   {
      if (startedAppservers.containsKey(appserverName))
      {
         return true;
      }

      MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
      
      if (!msaDef.isAllServerSide())
      {
         LOG.severe("MSA appserver can't start without server-side-resources config of 'all'.");
         return false;
      }

      long startNano = System.nanoTime();

      // TODO: configured by ?
      int retryAttempts = 3;
      
      Throwable appserverStartException = null;
      boolean appserverStartSuccess = false;
      try
      {
         FutureTask<Boolean> agentStartupTask = new FutureTask<>(new Callable<Boolean>()
         {
            private Boolean startAppserver(int retryAttempts)
            throws Exception
            {
               try
               {
                  MultiSessionAppserver msaAppserver = new MultiSessionAppserver(msaDef,
                                                                                   connections::values,
                                                                                   REMOVE_TIMEDOUT_CONN_FUNC);
                  startedAppservers.put(appserverName, msaAppserver);
                  connections.put(appserverName, msaAppserver.getAdminConnection());
                  return true;
               }
               catch (Throwable t)
               {
                  if (retryAttempts > 0)
                  {
                     return startAppserver(retryAttempts - 1);
                  }
                  throw t;
               }
            }
            
            @Override
            public Boolean call()
            throws Exception
            {
               return startAppserver(retryAttempts);
            }
         });

         agentStartupTask.run();
         appserverStartSuccess = agentStartupTask.get(msaDef.getAgentListenerTimeout(), TimeUnit.MILLISECONDS);
      }
      catch (Exception e)
      {
         appserverStartException = e.getCause();
      }

      long endMs = (System.nanoTime() - startNano) / 1000000;
      
      if (appserverStartSuccess)
      {
         LOG.log(Level.INFO, "Agent for appserver '%s' was started successfully (%dms)!", appserverName, endMs);
      }
      else
      {
         startedAppservers.remove(appserverName);
         
         LOG.log(Level.SEVERE,
                 appserverStartException,
                 "Agent for appserver '%s' could not be started (%dms)!",
                 appserverName,
                 endMs);
      }

      return false;
   }

   /**
    * Restart all agents in the specified appserver.
    *
    * @param    appserverName
    *           The appserver name to restart.
    */
   public void restart(String appserverName)
   {
      if (startedAppservers.containsKey(appserverName))
      {
         MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
         msaAppserver.terminateAppserver(MultiSessionAgent.ExitCode.EXPECTED_4020);
         clearApp(appserverName);
      }

      start(appserverName);
   }

   /**
    * Terminates the selected sessions of the appserver and returns the refreshed list of appserver agents.
    *
    * @param    appserverName
    *           The appserver name
    * @param    sessions
    *           The list of sessions to be terminated
    */
   public void terminateSessions(String appserverName, MultiSessionAgentSessionDef[] sessions)
   {
      if (!startedAppservers.containsKey(appserverName))
      {
         return;
      }

      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      Arrays.stream(sessions)
            .parallel()
            .forEach(sessionDef ->
                        msaAppserver.terminateSession(sessionDef.getAgentId(), 
                                                        sessionDef.getSessionId(), 
                                                        true,
                                                        false));
   }
   
   /**
    * Terminates the array of agents in the specified appserver.
    *
    * @param    appserverName
    *           The appserver name to restart.
    * @param    agentIds
    *           The array of agent ids to be terminated.
    */
   public void terminateAgents(String appserverName, int... agentIds)
   {
      if (!startedAppservers.containsKey(appserverName))
      {
         return;
      }
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      for (int agentId : agentIds)
      {
         LOG.info("Terminating MSA agent with id " + agentId + " from app " + appserverName);
         msaAppserver.terminateAgent(agentId, MultiSessionAgent.ExitCode.EXPECTED_4020);
      }
   }

   /**
    * Trim the agents in the specified appserver.
    *
    * @param    appserverName
    *           The appserver name to trim.
    */
   public void trim(String appserverName)
   {
      if (startedAppservers.containsKey(appserverName))
      {
         MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
         msaAppserver.removeIdleSessions(MultiSessionAgent.SINGLE_AGENT_ID);
      }
   }

   /**
    * Returns {@link AppServerInvocationResult} after initializing the specified proxy procedure, by 
    * calling the <code>execute</code> method.
    *
    * @param   hProc
    *          The procedure handle.
    * @param   result
    *          The invocation result container.
    * @param   connectionId
    *          The connection id.
    * @param   externalProcName
    *          The name of the remote external procedure.
    */
   public void obtainProxy(handle hProc,
                           AppServerInvocationResult result,
                           String connectionId,
                           String externalProcName)
   {
      // this request id is needed in the consecutive proxy calls
      int requestId = AppServerManager.uniqueId();

      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.OBTAIN_PROXY,
                                                         connectionId, 
                                                         requestId,
                                                         DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT,
                                                         true);

      InvokeConfig invokeConfig = new InvokeConfig();
      invokeConfig.setProcedureHandle(hProc);
      if (!standardInvoke.determineConnection(invokeConfig, externalProcName))
      {
         return;
      }

      standardInvoke.exec(() -> {
         CoreAppserver.obtainProxy(hProc,
                                   result,
                                   standardInvoke.msaAppserver.getExports(),
                                   standardInvoke.msaDef.getSessionActivateProc(),
                                   standardInvoke.msaDef.getSessionDeactivateProc(),
                                   standardInvoke.invokeConnection::isBound,
                                   standardInvoke.invokeConnection.getId(),
                                   standardInvoke.agentId,
                                   standardInvoke.sessionId,
                                   externalProcName,
                                   SET_RUNNING_PROC_FUNC,
                                   BIND_PERSISTENT_FUNC,
                                   ADD_PROC_FUNC,
                                   DISCONNECT_FUNC);
      });
   }

   /**
    * Returns {@link AppServerInvocationResult} after initializing the specified proxy procedure, by 
    * calling the <code>execute</code> method.
    *
    * @param   connectionId
    *          The connection id.
    * @param   requestId
    *          The request id.
    * @param   externalProcResId
    *          The external procedure resource id.
    * @param   agentId
    *          The agent id where this remote procedure was created.
    * @param   transactionDistinct
    *          Flag indicating if the TRANSACTION DISTINCT clause is in effect.
    * @param   modes
    *          A string representation of the modes of each parameter. May be <code>null</code>
    * @param   args
    *          The procedure's arguments.
    *
    * @return  See above.
    */
   public AppServerInvocationResult initializeProxy(String connectionId,
                                                    int requestId,
                                                    String externalProcResId,
                                                    int agentId,
                                                    boolean transactionDistinct,
                                                    String modes,
                                                    Object[] args)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INIT_PROXY,
                                                         connectionId,
                                                         requestId,
                                                         DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT,
                                                         true);

      if (!standardInvoke.determineConnection(externalProcResId))
      {
         return standardInvoke.result;
      }

      return standardInvoke.exec(() -> {
         ExternalProgramWrapper extProg = standardInvoke.invokeConnection.getProcWrapper(externalProcResId);

         CoreAppserver.initializeProxy(standardInvoke.result,
                                       externalProcResId,
                                       extProg,
                                       standardInvoke.invokeConnection.getId(),
                                       agentId,
                                       getSessionId(),
                                       requestId,
                                       transactionDistinct,
                                       modes,
                                       args,
                                       SET_RUNNING_PROC_FUNC,
                                       REMOVE_PROC_FUNC,
                                       UNBIND_PERSISTENT_FUNC,
                                       standardInvoke.invokeConnection::isBound,
                                       standardInvoke.msaDef.getSessionDeactivateProc());
      });
   }

   /**
    * Delete the remote procedure, specified by the resource id. This must be called at the same
    * time when the local proxy procedure is deleted.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    procResId
    *           The ID of a remote procedure which needs to be deleted.
    * @param    agentId
    *           The agent ID where this remote procedure was created.
    *
    * @return   <code>true</code> if the remote procedure was deleted.
    */
   public boolean deleteProcedure(String connectionId, String procResId, int agentId)
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection == null || connection.isDisconnecting())
      {
         return false;
      }

      String appserverName = connection.getAppserverName();
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      if (msaAppserver == null)
      {
         return false;
      }

      MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
      MultiSessionClientConnection invokeConnection = connection.findConnectionFor(procResId);
      
      CancellableCompositeTask<AppServerInvocationResult> invokeTask =
         new CancellableCompositeTask<>(new MsaCancellableCallable()
         {
            @Override
            public AppServerInvocationResult call()
            throws Exception
            {
               int sessionId = reserveSession(msaAppserver, invokeConnection);
               
               msaAppserver.runInTopScope(invokeConnection, sessionId, () ->
               {
                  invokeConnection.deletePersistentProc(procResId);

                  if (invokeConnection.isBound())
                  {
                     if (!invokeConnection.hasPersistentProc())
                     {
                        msaAppserver.unbind(invokeConnection, sessionId, true);
                     }

                     CoreAppserver.invokeDeactivate(msaDef.getSessionDeactivateProc(),
                                                    isConnectionBound(),
                                                    invokeConnection.getId(),
                                                    SET_RUNNING_PROC_FUNC);
                  }
               });
               return null;
            }
         }, MultiSessionTaskType.DELETE);

      AppServerInvocationResult result = new AppServerInvocationResult();
      handleExec(invokeTask,
                 result,
                 invokeConnection,
                 msaDef.getRequestWaitTimeout(),
                 msaDef.getSessionExecutionTimeLimit());
      
      return result.getError() != null;
   }
   
   /**
    * Delete the remote object, specified by the resource ID. This must be called at the same
    * time when the local proxy object is deleted.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    objResId
    *           The ID of a remote object which needs to be deleted.
    * @param    agentId
    *           The agent ID where this remote procedure was created.
    *
    * @return   <code>true</code> if the remote object was deleted.
    */
   public boolean deleteObject(String connectionId, String objResId, int agentId)
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection == null || connection.isDisconnecting())
      {
         return false;
      }

      String appserverName = connection.getAppserverName();
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      if (msaAppserver == null)
      {
         return false;
      }

      MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
      MultiSessionClientConnection invokeConnection = connection.findConnectionFor(objResId);

      CancellableCompositeTask<AppServerInvocationResult> invokeTask =
         new CancellableCompositeTask<>(new MsaCancellableCallable()
         {
            @Override
            public AppServerInvocationResult call()
            throws Exception
            {
               int sessionId = reserveSession(msaAppserver, invokeConnection);

               msaAppserver.runInTopScope(invokeConnection, sessionId, () ->
               {
                  invokeConnection.deletePersistentObject(objResId);

                  if (invokeConnection.isBound())
                  {
                     if (!invokeConnection.hasPersistentProc())
                     {
                        msaAppserver.unbind(invokeConnection, sessionId, true);
                     }

                     CoreAppserver.invokeDeactivate(msaDef.getSessionDeactivateProc(),
                                                    invokeConnection.isBound(),
                                                    invokeConnection.getId(),
                                                    SET_RUNNING_PROC_FUNC);
                  }
               });
               return null;
            }
         }, MultiSessionTaskType.DELETE);

      AppServerInvocationResult result = new AppServerInvocationResult();
      handleExec(invokeTask,
                 result,
                 invokeConnection,
                 msaDef.getRequestWaitTimeout(),
                 msaDef.getSessionExecutionTimeLimit());

      return result.getError() != null;
   }

   /**
    * Send a STOP condition to all the requests on the specified connection and clear the queue.
    *
    * @param    connectionId
    *           The appserver connection id. 
    */
   public void sendStop(String connectionId)
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection == null || connection.isDisconnecting())
      {
         return;
      }
      
      connection.cancelTasks();
   }
   
   /**
    * Establish a new appserver connection. This will invoke any connect procedure for this 
    * appserver, to which the given <code>user</code>, <code>pwd</code> and <code>serverInfo</code>
    * parameters will be passed.
    * <p>
    * The appserver connection must be configured to start under the given P2J-level subject ID.
    *
    * @param    appserverName
    *           The name of the target appserver.
    * @param    sessionFree
    *           Flag indicating if the connection request is for a session-free operating mode.
    * @param    user
    *           The first parameter to pass to the connect procedure, if any.
    * @param    pwd
    *           The second parameter to pass to the connect procedure, if any.
    * @param    serverInfo
    *           The third parameter to pass to the connect procedure, if any.
    * @param    requestInfo
    *           The remote {@link com.goldencode.p2j.oo.lang.OerequestInfo} details, as set by
    *           {@link com.goldencode.p2j.oo.lang.OerequestInfo#toArray()}.
    * @param    webServiceType
    *           The type of web service the worker is used for.
    *
    * @return   An {@link AppServerInvocationResult} instance with any result or caught errors.
    */
   public AppServerInvocationResult connect(String appserverName,
                                            boolean sessionFree,
                                            String user,
                                            String pwd,
                                            String serverInfo,
                                            Object[] requestInfo,
                                            String webServiceType)
   {
      AppServerInvocationResult result = new AppServerInvocationResult();
      
      if (!startedAppservers.containsKey(appserverName))
      {
         result.setError(new NumberedException("Application server connect failure.", 5468));
         return result;
      }
      
      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      
      if (!msaAppserver.isRunning())
      {
         result.setError(new NumberedException("Application server connect failure.", 5468));
         return result;
      }
      
      MultiSessionClientConnection connection = createNewClientConnection(appserverName,
                                                                          null,
                                                                          sessionFree,
                                                                          webServiceType != null);
      String connectionId = connection.getId();

      MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
      
      CancellableCompositeTask<AppServerInvocationResult> connectTask =
         new CancellableCompositeTask<>(new MsaCancellableCallable()
         {
            @Override
            public AppServerInvocationResult call()
            throws Exception
            {
               if (!sessionFree)
               {
                  msaAppserver.log(
                     "Application Server connected with connection id: " + connectionId + ". (8358)",
                     LoggingLevel.BASIC,
                     LoggingEntryType.DB_Connects,
                     LoggingExecEnv.AS_ADMIN);
               }

               String connectProc = msaDef.getSessionConnectProc();
               if (sessionFree || connectProc == null)
               {
                  result.setConnectionID(connectionId);
                  result.setConnectionModel(AppServerSessionModel.SESSION_FREE);
                  return result;
               }

               int sessionId = reserveSession(msaAppserver, connection);

               int threadId = (int) Thread.currentThread().getId();

               result.setAgentId(MultiSessionAgent.SINGLE_AGENT_ID);
               result.setConnectionID(connectionId);
               result.setConnectionModel(AppServerSessionModel.SESSION_MANAGED);
               result.setSessionId(sessionId);
               result.setThreadId(threadId);

               if (isCancelled() || sessionId == MultiSessionAppserver.INVALID_SESSION_ID)
               {
                  String errMsg = getCancelCause() == CancelCause.SEND_STOP ? "" : ERR_NO_CONNECTIONS;
                  result.setError(new ErrorConditionException(errMsg));
                  return result;
               }

               SET_RUNNING_PROC_FUNC.accept(connectionId, connectProc);
               
               msaAppserver.runInTopScope(result, requestInfo, connection, sessionId, connectProc, () ->
               {
                  ControlFlowOps.invokeExternalProcedure(new character(connectProc),
                                                         false,
                                                         null,
                                                         false,
                                                         null,
                                                         "III",
                                                         new character(user),
                                                         new character(pwd),
                                                         new character(serverInfo));
               }, getIsRunningSupplier(), getCancelledSupplier());

               SET_RUNNING_PROC_FUNC.accept(connectionId, null);
               
               return result;
            }
         }, MultiSessionTaskType.CONNECT);
      
      return handleExec(connectTask,
                        result,
                        connection,
                        msaDef.getRequestWaitTimeout(),
                        msaDef.getSessionExecutionTimeLimit());
   }
   
   /**
    * Disconnect and cleanup the appserver connection.
    *
    * @param    connectionId
    *           The connection id.
    * @param    shutdownBoundSession
    *           Flag to indicate the bound MSA session should be shut down.
    *           
    * @return   <code>true</code> if an active connection has been disconnected.
    */         
   public boolean disconnect(String connectionId, boolean shutdownBoundSession)
   {
      MultiSessionClientConnection connection = connections.get(connectionId);
      if (connection == null || connection.isDisconnecting())
      {
         return false;
      }

      connection.setDisconnecting(true);
      connection.disableLegacyErrors(false);
      
      String appserverName = connection.getAppserverName();
      if (!startedAppservers.containsKey(appserverName))
      {
         connections.remove(connectionId);
         return false;
      }

      MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
      if (!msaAppserver.isRunning())
      {
         connections.remove(connectionId);
         return false;
      }

      MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
      String disconnectProc = msaDef.getSessionDisconnProc();

      int sessionId = MultiSessionAppserver.INVALID_SESSION_ID;
      try
      {
         sessionId = msaAppserver.reserveSession(connection).get();
      }
      catch (Exception e)
      {
         LOG.info("Reserving MSA session for disconnect failed.", e);
      }

      // TODO: OE waits for the currently running async proc and then disconnects, clearing next in queue
      
      if (sessionId == MultiSessionAppserver.INVALID_SESSION_ID)
      {
         LOG.warning(ERR_NO_CONNECTIONS);
         clearConnections(msaAppserver, connection, shutdownBoundSession);
         return false;
      }

      final int finalSessionId = sessionId;
      try
      {
         msaAppserver.runInTopScope(null, null, connection, sessionId, disconnectProc, () ->
         {
            SET_RUNNING_PROC_FUNC.accept(connectionId, disconnectProc);
            
            if (!connection.isSessionFree() && disconnectProc != null)
            {
               ControlFlowOps.invokeExternalProcedure(new character(disconnectProc),
                                                      false,
                                                      null,
                                                      false,
                                                      null,
                                                      null);
            }
            SET_RUNNING_PROC_FUNC.accept(connectionId, null);

            for (MultiSessionClientConnection childConnection : connection.getChildConnections())
            {
               Integer boundSessionId = childConnection.getBoundSessionId();
               boolean doesContextExist = 
                  MultiSessionAppserverSecurityManager.getInstance()
                                                      .hasContext(appserverName, boundSessionId);
               if (boundSessionId == null || finalSessionId == boundSessionId || !doesContextExist)
               {
                  msaAppserver.deleteConnection(childConnection, false);
               }
               else
               {
                  try
                  {
                     Runnable runnable = () -> {
                        msaAppserver.deleteConnection(childConnection, shutdownBoundSession);
                     };
                     msaAppserver.runInSessionContext(connection,
                                                      boundSessionId,
                                                      null,
                                                      runnable,
                                                      null,
                                                      null,
                                                      null);
                  }
                  catch (Exception e)
                  {
                     if (LOG.isLoggable(Level.FINE))
                     {
                        LOG.log(Level.FINE, e, "Disconnect error: ");
                     }
                  }
               }
            }

            msaAppserver.deleteConnection(connection, shutdownBoundSession);
         }, null, null);
      }
      catch (Exception e)
      {
         if (LOG.isLoggable(Level.FINE))
         {
            LOG.log(Level.FINE, e, "Disconnect error: ");
         }
      }

      clearConnections(msaAppserver, connection, shutdownBoundSession);
      
      return true;
   }
   
   /**
    * Invoke a remote external procedure, internal procedure or user-defined function. The 
    * validation for the program name and arguments will be done on the remote side.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    cfg
    *           The invoke configuration.
    *
    * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
    *           arguments and any returned value.
    */
   public AppServerInvocationResult invoke(String connectionId, int requestId, InvokeConfig cfg)
   {
      return invoke(connectionId, requestId, cfg, null, -1);
   }

   /**
    * Invoke a remote external procedure, internal procedure or user-defined function. The 
    * validation for the program name and arguments will be done on the remote side.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    cfg
    *           The invoke configuration.
    * @param    inProcResId
    *           In handle resource id.
    * @param    execTimeout
    *           The maximum allowed time for this invocation to complete. In milliseconds.
    *           Use <code>0</code> to disable the timeout.
    *
    * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
    *           arguments and any returned value.
    */
   public AppServerInvocationResult invoke(String connectionId,
                                           int requestId,
                                           InvokeConfig cfg,
                                           String inProcResId,
                                           long execTimeout)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                         connectionId, requestId, execTimeout, false);

      if (!standardInvoke.determineConnection(cfg, inProcResId))
      {
         return standardInvoke.result;
      }

      return standardInvoke.exec(() -> {
         if (standardInvoke.isPersistentProcRun)
         {
            CoreAppserver.runPersistent(standardInvoke.result,
                                        true,
                                        standardInvoke.msaAppserver.getExports(),
                                        standardInvoke.msaDef.getSessionActivateProc(),
                                        standardInvoke.msaDef.getSessionDeactivateProc(),
                                        standardInvoke.invokeConnection::isBound,
                                        standardInvoke.invokeConnection.getId(),
                                        MultiSessionAgent.SINGLE_AGENT_ID,
                                        standardInvoke.threadId,
                                        standardInvoke.sessionId,
                                        cfg,
                                        SET_RUNNING_PROC_FUNC,
                                        BIND_PERSISTENT_FUNC,
                                        UNBIND_PERSISTENT_FUNC,
                                        ADD_PROC_FUNC,
                                        DISCONNECT_FUNC);
         }
         else
         {
            CoreAppserver.invoke(standardInvoke.result,
                                 true,
                                 true,
                                 standardInvoke.msaAppserver.getExports(),
                                 standardInvoke.msaDef.getSessionActivateProc(),
                                 standardInvoke.msaDef.getSessionDeactivateProc(),
                                 standardInvoke.invokeConnection::isBound,
                                 standardInvoke.invokeConnection.getId(),
                                 MultiSessionAgent.SINGLE_AGENT_ID,
                                 standardInvoke.threadId,
                                 standardInvoke.sessionId,
                                 cfg,
                                 inProcResId,
                                 SET_RUNNING_PROC_FUNC,
                                 REMOVE_PROC_FUNC,
                                 DISCONNECT_FUNC);
         }
      });
   }

   /**
    * Perform a pseudo-dynamic call where the target procedure doesn't receive the parameter as arguments at
    * the Java method definition, but instead they are managed via {@link LegacyOpenClientCaller} APIs.
    * <p>
    * This allows the target program to act as a controller, where it can prepare the arguments, perform
    * security checks, etc, before dispatching the call to the real target, which can be resolved from the
    * arguments or in some other way.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    execTimeout
    *           The maximum allowed time for this invocation to complete.  In milliseconds.
    *           Use <code>0</code> to disable the timeout.
    * @param    procedure
    *           The target external program.  Must have no parameters defined.
    * @param    argTypes
    *           The argument types, one of the e.g. {@link LegacyJavaAppserverApi#integer()} values.
    * @param    modes
    *           The argument modes, one of the (I)NPUT, (O)UTPUT or INP(U)T-OUTPUT values.
    * @param    args
    *           The actual arguments.
    *
    * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT arguments
    *           and any returned value.
    */
   public AppServerInvocationResult invokeWithArgs(String connectionId,
                                                   int requestId,
                                                   long execTimeout,
                                                   String procedure,
                                                   int[] argTypes,
                                                   String modes,
                                                   Object[] args)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                         connectionId, requestId, execTimeout, false);

      return standardInvoke.exec(() -> {
         CoreAppserver.invokeWithArgs(standardInvoke.result,
                                      standardInvoke.invokeConnection.getId(),
                                      standardInvoke.msaDef.getSessionActivateProc(),
                                      standardInvoke.msaDef.getSessionDeactivateProc(),
                                      standardInvoke.invokeConnection::isBound,
                                      SET_RUNNING_PROC_FUNC,
                                      DISCONNECT_FUNC,
                                      procedure,
                                      argTypes,
                                      modes,
                                      args);
      });
   }
   
   /**
    * Create a new instance of the specified qualified class name.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    execTimeout
    *           The maximum allowed time for this invocation to complete.  In milliseconds.
    *           Use <code>0</code> to disable the timeout.
    * @param    name
    *           The legacy qualified class name for the legacy class.
    * @param    modes
    *           A string representation of the modes of each parameter. May be <code>null</code>
    * @param    args
    *           The constructor's arguments.
    *
    * @return   A {@link AppServerInvocationResult} instance, which holds the code for the remote
    *           object, all updated OUTPUT/INPUT-OUTPUT arguments and any returned value.
    */
   public AppServerInvocationResult newInstance(String connectionId,
                                                int requestId,
                                                long execTimeout,
                                                character name,
                                                String modes,
                                                Object[] args)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.NEW_INSTANCE, 
                                                         connectionId, requestId, execTimeout, true);

      if (!standardInvoke.determinePseudoPersistentConnection())
      {
         return standardInvoke.result;
      }
         
      return standardInvoke.exec(() -> {
         CoreAppserver.newInstance(standardInvoke.result,
                                   standardInvoke.invokeConnection.getId(),
                                   MultiSessionAgent.SINGLE_AGENT_ID,
                                   standardInvoke.sessionId,
                                   standardInvoke.msaDef.getSessionActivateProc(),
                                   standardInvoke.msaDef.getSessionDeactivateProc(),
                                   standardInvoke.invokeConnection::isBound,
                                   SET_RUNNING_PROC_FUNC,
                                   BIND_PERSISTENT_FUNC,
                                   UNBIND_PERSISTENT_FUNC,
                                   ADD_PROC_FUNC,
                                   DISCONNECT_FUNC,
                                   name,
                                   modes,
                                   args);
      });
   }
   
   /**
    * Invoke a method in a remotely instantiated class.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    execTimeout
    *           The maximum allowed time for this invocation to complete.  In milliseconds.
    *           Use <code>0</code> to disable the timeout.
    * @param    returnValue
    *           Flag indicating if the return value is required.
    * @param    name
    *           The legacy 4GL name for the class method.
    * @param    resId
    *           The code of a remotely instantiated object in which to invoke the method.
    * @param    agentId
    *           The agent ID where this remote procedure was created.
    * @param    modes
    *           A string representation of the modes of each parameter. May be <code>null</code>
    * @param    args
    *           The method's arguments.
    *
    * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
    *           arguments and any returned value.
    */
   public AppServerInvocationResult invokeMethod(String connectionId,
                                                 int requestId,
                                                 long execTimeout,
                                                 boolean returnValue,
                                                 character name,
                                                 String resId,
                                                 int agentId,
                                                 String modes,
                                                 Object[] args)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE, 
                                                         connectionId, requestId, execTimeout, true);
      
      if (!standardInvoke.determineConnection(resId))
      {
         return standardInvoke.result;
      }

      return standardInvoke.exec(() -> {
         CoreAppserver.invokeMethod(standardInvoke.result,
                                    true,
                                    standardInvoke.invokeConnection.getId(),
                                    standardInvoke.invokeConnection::isBound,
                                    SET_RUNNING_PROC_FUNC,
                                    returnValue,
                                    name,
                                    resId,
                                    modes,
                                    args);
      });
   }

   /**
    * Invoke the given Java method directly. Used by {@link JavaRestService}.
    * <p>
    * All arguments are sent by references: if it was marked as an OUTPUT or INPUT-OUTPUT, any change to that
    * instance will be reflected back to the caller.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    execTimeout
    *           The maximum allowed time for this invocation to complete.  In milliseconds.
    *           Use <code>0</code> to disable the timeout.
    * @param    cfg
    *           The configuration with the Java method to be invoked.
    *
    * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
    *           arguments and any returned value.
    */
   public AppServerInvocationResult invokeJavaMethod(String connectionId,
                                                     int requestId,
                                                     int execTimeout,
                                                     JavaInvokeConfig cfg)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                         connectionId, requestId, execTimeout, false);
      
      return standardInvoke.exec(() -> {
         CoreAppserver.invokeJavaMethod(standardInvoke.result,
                                        standardInvoke.invokeConnection.getId(),
                                        true,
                                        standardInvoke.msaDef.getSessionActivateProc(),
                                        standardInvoke.msaDef.getSessionDeactivateProc(),
                                        standardInvoke.invokeConnection::isBound,
                                        SET_RUNNING_PROC_FUNC,
                                        DISCONNECT_FUNC,
                                        cfg);
      });
   }
   
   /**
    * Handle a generic web service request.  This exposes the Jetty request and response objects
    * to the Agent which will process it, so it is required for the appserver to be local.
    *
    * @param    connectionId
    *           The appserver connection id. 
    * @param    requestId
    *           The ID of the request.
    * @param    execTimeout
    *           The maximum allowed time for this invocation to complete.  In milliseconds.
    *           Use <code>0</code> to disable the timeout.
    * @param    handler
    *           The {@link com.goldencode.p2j.oo.web.WebHandler} implementation.
    * @param    basepath
    *           The service basepath.
    * @param    paths
    *           The split paths for this handler.
    * @param    token
    *           When not null, it represents the token of a FWD context created for an authenticated and 
    *           authorized web service call.  The API call will be performed in that context.
    * @param    target
    *           The request original target.
    * @param    request
    *           The Jetty request.
    * @param    response
    *           The Jetty response.
    *
    * @return   A {@link AppServerInvocationResult} instance with any error thrown during the web service
    *           processing.
    */
   public AppServerInvocationResult handleWebService(String connectionId,
                                                     int requestId,
                                                     long execTimeout,
                                                     Class<? extends WebHandler> handler,
                                                     String basepath,
                                                     String[] paths,
                                                     String token,
                                                     String target,
                                                     HttpServletRequest request,
                                                     HttpServletResponse response)
   {
      StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.WEB,
                                                         connectionId, requestId, execTimeout, false);

      return standardInvoke.exec(() -> {
         CoreAppserver.handleWebService(standardInvoke.result, 
                                        standardInvoke.invokeConnection.getId(),
                                        handler,
                                        basepath, 
                                        paths,
                                        target,
                                        request, 
                                        response, 
                                        SET_RUNNING_PROC_FUNC);
      });
   }

   /**
    * Disable legacy OO errors thrown on this connection.
    *
    * @param    connectionId
    *           The appserver connection id. 
    */
   public void disableLegacyErrors(String connectionId)
   {
      MultiSessionClientConnection connection = getConnection(connectionId);
      if (connection != null)
      {
         connection.disableLegacyErrors(true);
      }
   }

   /**
    * Check if the current {@link WorkArea#connection connection} should suppress legacy OO errors.
    *
    * @return   See above.
    */
   public boolean hasLegacyErrorsDisabled()
   {
      MultiSessionClientConnection currentConnection = local.get().connection;
      return currentConnection != null && currentConnection.hasLegacyErrorsDisabled();
   }
   
   /**
    * Get the SERVER-CONNECTION-BOUND-REQUEST attribute of this session.
    *
    * @return   See above.
    */
   public static boolean isServerConnectionBoundRequest()
   {
      WorkArea wa = local.get();
      if (!isMsaContext.get() || wa.connection == null || wa.connection.isSessionFree())
      {
         return false;
      }

      return wa.connection.getServerConnectionBoundRequest();
   }

   /**
    * Set the SERVER-CONNECTION-BOUND-REQUEST attribute of this session.
    *
    * @param   bind
    *          The bound state.
    */
   public static void setServerConnectionBoundRequest(boolean bind)
   {
      WorkArea wa = local.get();
      if (!isMsaContext.get())
      {
         return;
      }

      String appserverName = wa.connection.getAppserverName();
      MultiSessionAppserver msaAppserver = MultiSessionAppserverManager
         .getInstance().startedAppservers.get(appserverName);
      if (msaAppserver == null)
      {
         return;
      }

      if (wa.isStartupProc)
      {
         LegacyLogOps.logMgr().writeMessage(
            "It is not allowed to set SERVER-CONNECTION-BOUND-REQUEST to 'yes' " +
               "in the startup  procedure of a stateless server. (8024)",
            LoggingEntryType.ASPlumbing,
            null,
            LoggingSubsys.EMPTY,
            LoggingLevel.ERROR);

         // TODO: replace with proper exception instead of logging separate
         throw new ErrorConditionException(
            "It is not allowed to set SERVER-CONNECTION-BOUND-REQUEST to 'yes' " +
               "in the startup  procedure of a stateless server.", 8024);
      }

      if (wa.connection.isSessionFree() || wa.connection == null || wa.connection.isDisconnecting())
      {
         return;
      }
      
      wa.connection.setServerConnectionBoundRequest(bind);

      if (bind)
      {
         msaAppserver.bind(wa.connection, wa.sessionId, false);
      }
      else
      {
         msaAppserver.unbind(wa.connection, wa.sessionId, false);
      }
   }

   /**
    * Sets details of the running task to the current context and prepares the OeRequestInfo.
    *
    * @param    connection
    *           The appserver connection. 
    * @param    agentId
    *           The ID of the agent.
    * @param    sessionId
    *           The ID of the session.
    * @param    isInitialSetup
    *           <code>true</code> if currently running agent/session initialization.
    * @param    debugStackTrace
    *           Stacktrace of the original thread of the MsaTask. Used for debugging.
    */
   void setMsaContext(MultiSessionClientConnection connection,
                      Integer agentId,
                      Integer sessionId,
                      boolean isInitialSetup,
                      List<StackTraceElement[]> debugStackTrace)
   {
      isMsaContext.set(true);
      
      WorkArea wa = local.get();
      wa.connection = connection;
      wa.agentId = agentId;
      wa.sessionId = sessionId;
      wa.isStartupProc = isInitialSetup;
      wa.debugStackTrace = debugStackTrace;
      
      if (isInitialSetup)
      {
         CoreAppserver.setOeRequestInfo(null, true, null, null, null);
      }
      else
      {
         CoreAppserver.setOeRequestInfo(null,
                                        true,
                                        agentId,
                                        (int) Thread.currentThread().getId(),
                                        sessionId);
      }
   }

   /**
    * Resets the details of the running task in the current context and resets the OeRequestInfo.
    */
   void resetMsaContext()
   {
      WorkArea wa = local.get();
      wa.agentId = null;
      wa.connection = null;
      wa.sessionId = null;
      wa.isStartupProc = false;
      wa.debugStackTrace = null;

      CoreAppserver.setOeRequestInfo(null, true, null, null, null);
   }
   
   /**
    * Clears references to an appserver from the manager.
    *
    * @param    appserverName
    *           The appserver name.
    */
   void clearApp(String appserverName)
   {
      startedAppservers.remove(appserverName);
      connections.entrySet()
                 .removeIf(entry -> entry.getValue().getAppserverName().equals(appserverName));
   }

   /**
    * Clears references to a connection from the MSA manager and the appserver.
    *
    * @param    msaAppserver
    *           The appserver.
    * @param    primaryConnection
    *           The primary connection to be cleared.
    * @param    shutdownBoundSession
    *           <code>true</code> if the bound MSA session should be shut down.
    */
   private void clearConnections(MultiSessionAppserver msaAppserver,
                                 MultiSessionClientConnection primaryConnection,
                                 boolean shutdownBoundSession)
   {
      for (MultiSessionClientConnection childConnection : primaryConnection.getChildConnections())
      {
         clearConnection(msaAppserver, childConnection.getId(), shutdownBoundSession);
      }

      clearConnection(msaAppserver, primaryConnection.getId(), shutdownBoundSession);
   }

   /**
    * Clears references to a connection from the MSA manager and the appserver.
    *
    * @param    msaAppserver
    *           The appserver.
    * @param    connectionId
    *           The id of the connection to be cleared.
    * @param    shutdownBoundSession
    *           <code>true</code> if the bound MSA session should be shut down.
    */
   private void clearConnection(MultiSessionAppserver msaAppserver,
                                String connectionId,
                                boolean shutdownBoundSession)
   {
      connections.remove(connectionId);
      msaAppserver.clearConnection(connectionId, shutdownBoundSession);
   }
   
   /**
    * Returns a newly created client connection.
    *
    * @param    appserverName
    *           The appserver name.
    * @param    parentConnection
    *           The parent connection if such exists.
    * @param    isSessionFree
    *           <code>true</code> if the connection runs in Session-free session model.
    *           
    * @return   See above.
    */
   private MultiSessionClientConnection createNewClientConnection(String appserverName,
                                                                  MultiSessionClientConnection parentConnection,
                                                                  boolean isSessionFree)
   {
      return createNewClientConnection(appserverName, parentConnection, isSessionFree, false);
   }

   /**
    * Returns a newly created client connection.
    *
    * @param    appserverName
    *           The appserver name.
    * @param    parentConnection
    *           The parent connection if such exists.
    * @param    isSessionFree
    *           <code>true</code> if the connection runs in Session-free session model.
    * @param    isForWebService
    *           The connection is used by a web service.
    *
    * @return   See above.
    */
   private MultiSessionClientConnection createNewClientConnection(String appserverName,
                                                                  MultiSessionClientConnection parentConnection,
                                                                  boolean isSessionFree,
                                                                  boolean isForWebService)
   {
      String newConnectionId = createConnectionId(appserverName);
      MultiSessionClientConnection connection = new MultiSessionClientConnection(newConnectionId,
                                                                                 parentConnection,
                                                                                 appserverName,
                                                                                 isSessionFree,
                                                                                 isForWebService);
      connections.put(newConnectionId, connection);
      return connection;
   }

   /**
    * Creates a MSA connection id with 80 random uppercase letters and digits symbols + "." + 
    * appserverName.
    *
    * @param    appserverName
    *           The appserver name.
    *
    * @return   The connection id.
    */
   private String createConnectionId(String appserverName)
   {
      Random randomGenerator = new Random();
      StringBuilder sb = new StringBuilder();

      for (int i = 0; i < CONNECTION_ID_RANDOM_CHAR_COUNT; i++)
      {
         int index = randomGenerator.nextInt(CONNECTION_ID_CHARS.length);
         char randomChar = (char) CONNECTION_ID_CHARS[index];
         sb.append(randomChar);
      }

      sb.append(".").append(appserverName);

      return sb.toString();
   }

   /**
    * Generic result handling for MSA tasks.
    *
    * @param    task
    *           The wrapper for the MSA task.
    * @param    result
    *           The instance of the result object to be used.
    * @param    connection
    *           The client connection.
    * @param    requestWaitTimeout
    *           The timeout for cancelling the request before it starts running (the time for finding an 
    *           MSA session and a free worker thread).
    * @param    sessionExecutionTimeLimit
    *           The time limit for running the request.
    *
    * @return   A {@link AppServerInvocationResult} instance with any error thrown during the execution.
    */
   private AppServerInvocationResult handleExec(CancellableCompositeTask<AppServerInvocationResult> task,
                                                AppServerInvocationResult result,
                                                MultiSessionClientConnection connection,
                                                int requestWaitTimeout,
                                                int sessionExecutionTimeLimit)
   {
      connection.addTask(task);
      try
      {
         if (requestWaitTimeout > 0)
         {
            Runnable timeoutRunnable = () ->
            {
               synchronized (task.getRunningSupplier())
               {
                  if (!task.getRunningSupplier().get())
                  {
                     task.cancel(true);
                  }
               }
            };
            TIMEOUT_EXECUTOR.schedule(timeoutRunnable, requestWaitTimeout, TimeUnit.MILLISECONDS);
         }

         task.run();

         return sessionExecutionTimeLimit == 0 ?
            task.get() :
            task.get(sessionExecutionTimeLimit, TimeUnit.SECONDS);
      }
      catch (Exception e)
      {
         if (e instanceof CancellationException)
         {
            String msg = task.getCancelCause() == CancelCause.REQUEST_TIMEOUT ?
               ERR_NO_CONNECTIONS :
               task.getCancelCause() == null ? "" : task.getCancelCause().toString();

            if (task.getCancelCause() == CancelCause.REQUEST_TIMEOUT)
            {
               LOG.warning(msg);
            }

            result.setError(new ErrorConditionException(msg));
         }
         else if (e instanceof TimeoutException)
         {
            // sessionExecutionTimeLimit
            task.sendStop();
            result.setError(new StopConditionException("Session execution timed out."));
         }
         else if (e instanceof ExecutionException)
         {
            Throwable cause = e.getCause();
            // the MsaTask Future throws nested ExecutionException
            result.setError(cause instanceof ExecutionException ? cause.getCause() : cause);
         }
         else
         {
            result.setError(e);
         }
         return result;
      }
      finally
      {
         connection.removeTask(task);
      }
   }

   /**
    * A class to wrap up the standard handling of invoke calls: connection management, creating and executing 
    * MSA task wrappers that reserve an MSA session and set up the invoke details in the result.
    */
   private class StandardInvoke
   {
      /** The task type. */
      private final MultiSessionTaskType taskType;

      /** The id of the main connection. */
      private final String mainConnectionId;

      /** The request id. */
      private final int requestId;

      /** The session execution timeout specified by the client. */
      private final long clientDrivenSessionExecutionTimeout;

      /** The result instance. */
      private final AppServerInvocationResult result;

      /** Flag to indicate the wrapper has already validated the connection before starting execution. */
      private final AtomicBoolean isConnectionValidationDone = new AtomicBoolean();

      /** The appserver definition. */
      private MultiSessionAppserverDefinition msaDef;

      /** The appserver itself. */
      private MultiSessionAppserver msaAppserver;

      /** The main connection. */
      private MultiSessionClientConnection mainConnection;

      /** The invoke connection. */
      private MultiSessionClientConnection invokeConnection;

      /** Flag to indicate if the invoke should run in persistent mode. */
      private boolean isPersistentProcRun;
         
      /** The agent id of the reserved session id. */
      private Integer agentId;
      
      /** The reserved session id. */
      private Integer sessionId;

      /** The worker thread id running the invoke. */
      private Integer threadId;

      /**
       * Private constructor of the standard invoke wrapper.
       *
       * @param    taskType
       *           The type of the invoke.
       * @param    connectionId
       *           The appserver connection id. 
       * @param    requestId
       *           The ID of the request.
       * @param    clientDrivenSessionExecutionTimeout
       *           The maximum allowed time for this invocation to complete. In milliseconds.
       *           Use <code>0</code> to disable the timeout.
       * @param    isPersistentProcRun
       *           <code>true</code> if the invoke should run in persistent mode.
       */
      private StandardInvoke(MultiSessionTaskType taskType,
                             String connectionId,
                             int requestId,
                             long clientDrivenSessionExecutionTimeout,
                             boolean isPersistentProcRun)
      {
         this.taskType = taskType;
         this.mainConnectionId = connectionId;
         this.requestId = requestId;
         this.clientDrivenSessionExecutionTimeout = clientDrivenSessionExecutionTimeout;
         this.isPersistentProcRun = isPersistentProcRun;
         this.result = new AppServerInvocationResult();
         this.agentId = MultiSessionAgent.SINGLE_AGENT_ID;
      }

      /**
       * Validates if the main connection is running on a live appserver.
       *
       * @param    isInCall
       *           Flag to indicate if the invoke is RUN IN handle call, or method invoke.
       * 
       * @return   <code>true</code> if the main connection is found on a running appserver.
       */
      private boolean validateMainConnection(boolean isInCall)
      {
         mainConnection = connections.get(mainConnectionId);
         if (mainConnection == null || mainConnection.isDisconnecting())
         {
            boolean isReestablished = false;
            synchronized (timedOutConnections)
            {
               if (timedOutConnections.containsKey(mainConnectionId))
               {
                  MultiSessionClientConnection timedOutConn = timedOutConnections.remove(mainConnectionId);
                  if (timedOutConn.isSessionFree() && !isInCall)
                  {
                     mainConnection = createNewClientConnection(timedOutConn.getAppserverName(),
                                                                null,
                                                                timedOutConn.isSessionFree(),
                                                                timedOutConn.isForWebService());
                     mainConnection.disableLegacyErrors(timedOutConn.hasLegacyErrorsDisabled());
                     isReestablished = true;
                  }
                  else
                  {
                     result.setError(
                        new ErrorConditionException("Failure reading response from Application Server.",
                                                    14810), 
                        timedOutConn.hasLegacyErrorsDisabled());
                     return false;
                  }
               }
            }
            if (!isReestablished)
            {
               result.setError(new ErrorConditionException("Server is not connected.", 5537));
               return false;
            }
         }

         String appserverName = mainConnection.getAppserverName();
         if (!startedAppservers.containsKey(appserverName))
         {
            result.setError(new NumberedException("Application server connect failure.", 5468));
            return false;
         }

         msaDef = MultiSessionAppserverDefinition.get(appserverName);
         msaAppserver = startedAppservers.get(appserverName);
         if (!msaAppserver.isRunning())
         {
            result.setError(new NumberedException("Application server connect failure.", 5468));
            return false;
         }
         return true;
      }

      /**
       * Validates the main connection exists on a live appserver, if this is not already done, and determines
       * the invoke connection.
       *
       * @param    cfg
       *           The invoke configuration.
       * @param    inProcResId
       *           The resource id of the in-handle.
       *           
       * @return   <code>true</code> if the invoke connection is found on a running appserver.
       */
      private boolean determineConnection(InvokeConfig cfg, String inProcResId)
      {
         isConnectionValidationDone.set(true);

         boolean isInCall = cfg.getInHandle() != null || inProcResId != null;
         
         if (!validateMainConnection(isInCall))
         {
            return false;
         }
         
         String appserverName = msaDef.getAppserverName();
         isPersistentProcRun = !cfg.isClass() && cfg.getProcedureHandle() != null;

         if (isPersistentProcRun && mainConnection.isSessionFree())
         {
            // in Session-free persistent procs create their own client connections to bind
            invokeConnection = createNewClientConnection(appserverName, mainConnection, true);
            mainConnection.addChildConnection(invokeConnection);
         }
         else if (mainConnection.isSessionFree() && isInCall && !cfg.isSingleRun() && !cfg.isSingleton())
         {
            // in Session-free in-handle for persistent proc should switch the connection
            String persistentProcId = inProcResId != null ?
               inProcResId :
               ((ProxyProcedureWrapper) cfg.getInHandle().getResource()).getResourceId();
            invokeConnection = mainConnection.findConnectionFor(persistentProcId);
            if (invokeConnection == null)
            {
               result.setError(new ErrorConditionException("Server is not connected.", 5537),
                               mainConnection.hasLegacyErrorsDisabled());
               return false;
            }
         }
         else if (mainConnection.isSessionFree() && cfg.isAsynchronous())
         {
            // TODO: if it's the first async request running on the connection, use the main connection
            // in Session-free async procs create their own client connections to bind
            invokeConnection = createNewClientConnection(appserverName, mainConnection, true);
            mainConnection.addChildConnection(invokeConnection);
         }
         else
         {
            invokeConnection = mainConnection;
         }
         return true;
      }

      /**
       * Validates the main connection exists on a live appserver, if this is not already done, and determines
       * the invoke connection.
       *
       * @param    resId
       *           The external procedure resource id.
       *
       * @return   <code>true</code> if the invoke connection is found on a running appserver.
       */
      private boolean determineConnection(String resId)
      {
         isConnectionValidationDone.set(true);

         boolean isInCall = resId != null;
         if (!validateMainConnection(isInCall))
         {
            return false;
         }

         if (!isInCall)
         {
            invokeConnection = mainConnection;
            return true;
         }
         
         isPersistentProcRun = true;

         // in Session-free persistent procs should switch to their own connection
         invokeConnection = mainConnection.findConnectionFor(resId);
         if (invokeConnection == null)
         {
            throw new IllegalArgumentException("The procedure can not be accessed on this connection!");
            //result.setError(new NumberedException("Invalid or inappropriate handle value given to " +
            // "RUN...IN statement. Procedure '" + fileName + "':" + line + ".", 2128));
            //return false;
         }
         
         return true;
      }

      /**
       * Validates the main connection exists on a live appserver, if this is not already done, and determines
       * the invoke connection.
       *
       * @return   <code>true</code> if the invoke connection is found on a running appserver.
       */
      private boolean determinePseudoPersistentConnection()
      {
         isConnectionValidationDone.set(true);

         if (!validateMainConnection(false))
         {
            return false;
         }

         isPersistentProcRun = true;

         if (mainConnection.isSessionFree())
         {
            // in Session-free persistent procs create their own client connections to bind
            invokeConnection = createNewClientConnection(msaDef.getAppserverName(), mainConnection, true);
            mainConnection.addChildConnection(invokeConnection);
         }
         else
         {
            invokeConnection = mainConnection;
         }
         return true;
      }
      
      /**
       * Creates and executes a MSA task wrapper that reserves an MSA session and sets up the invoke 
       * details in the result.
       * 
       * @param    runnable
       *           The runnable to be executed on the worker.
       *           
       * @return   A {@link AppServerInvocationResult} instance, which holds the code for the remote
       *           object, all updated OUTPUT/INPUT-OUTPUT arguments and any returned value.
       */
      private AppServerInvocationResult exec(Runnable runnable)
      {
         if (!isConnectionValidationDone.get() && !determineConnection(null))
         {
            return result;
         }
         
         invokeConnection.setLastUseTime();

         CancellableCompositeTask<AppServerInvocationResult> invokeTask =
            new CancellableCompositeTask<>(new MsaCancellableCallable()
            {
               @Override
               public AppServerInvocationResult call()
               throws Exception
               {
                  sessionId = reserveSession(msaAppserver, invokeConnection);

                  if (isCancelled() || sessionId == MultiSessionAppserver.INVALID_SESSION_ID)
                  {
                     String errMsg = getCancelCause() == CancelCause.SEND_STOP ? "" : ERR_NO_CONNECTIONS;
                     result.setError(new ErrorConditionException(errMsg));
                     return result;
                  }

                  threadId = (int) Thread.currentThread().getId();

                  result.setAgentId(MultiSessionAgent.SINGLE_AGENT_ID);
                  result.setConnectionID(mainConnectionId);

                  if (!invokeConnection.getId().equals(mainConnectionId))
                  {
                     result.setChildConnectionID(invokeConnection.getId());
                  }
                  
                  result.setConnectionModel(invokeConnection.getModel());
                  result.setSessionId(sessionId);
                  result.setThreadId(threadId);
                  result.setRequestId(String.valueOf(requestId));

                  msaAppserver.runInSessionContext(invokeConnection,
                                                   sessionId,
                                                   result,
                                                   runnable,
                                                   getIsRunningSupplier(),
                                                   getCancelledSupplier(),
                                                   getTaskReceiver());

                  return result;
               }
            }, taskType, requestId);

         if (clientDrivenSessionExecutionTimeout > DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT &&
            LOG.isLoggable(Level.FINE))
         {
            LOG.fine("MSA request running with a custom timeout " + clientDrivenSessionExecutionTimeout + 
                        " milliseconds.");
         }

         return handleExec(invokeTask,
                           result,
                           invokeConnection,
                           msaDef.getRequestWaitTimeout(),
                           clientDrivenSessionExecutionTimeout > DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT ?
                              (int) clientDrivenSessionExecutionTimeout : 
                              msaDef.getSessionExecutionTimeLimit());
      }
   }
   
   /** Cancellable callable for the multi-session agent appservers. */
   private static abstract class MsaCancellableCallable
   extends CancellableCallable<AppServerInvocationResult>
   {
      /**
       * Manages the Future for reserving the session.
       * 
       * @param    msaAppserver
       *           The MSA appserver.
       * @param    invokeConnection
       *           The invoke connection.
       * 
       * @return   The id of the reserved session.
       * 
       * @throws   ExecutionException
       *           If the computation threw an exception      
       * @throws   InterruptedException
       *           If the current thread was interrupted while waiting.
       */
      Integer reserveSession(MultiSessionAppserver msaAppserver,
                             MultiSessionClientConnection invokeConnection)
      throws ExecutionException,
             InterruptedException
      {
         Future<Integer> reserveSessionFuture = msaAppserver.reserveSession(invokeConnection);
         setCurrentlyRunningFuture(reserveSessionFuture);
         int sessionId = reserveSessionFuture.get();
         setCurrentlyRunningFuture(null);
         return sessionId;
      }
   }

   /**
    * Stores global data relating to the state of the current MSA context.
    */
   private static class WorkArea
   {
      /** Flag to show a startup procedure is running. */
      private boolean isStartupProc;

      /** The id of the agent the current context is running in. */
      private Integer agentId;

      /** The id of the MSA session the current context is running in. */
      private Integer sessionId;

      /** The current client connection. */
      private MultiSessionClientConnection connection;

      /** Stacktrace of the original thread of the task. Used for debugging. */
      private List<StackTraceElement[]> debugStackTrace;

      /**
       * Constructor.
       */
      WorkArea()
      {
      }
   }
}