# MultiSessionAppserverManager.java

    /*
    ** Module   : MultiSessionAppserverManager.java
    ** Abstract : Manager for appserver with multi-session agents.
    **
    ** Copyright (c) 2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------Description----------------------------------
    ** 001 GBB 20250122 Created initial version.
    */
    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util.appserver;

    import com.goldencode.p2j.*;
    import com.goldencode.p2j.admin.*;
    import com.goldencode.p2j.main.*;
    import com.goldencode.p2j.oo.web.*;
    import com.goldencode.p2j.rest.*;
    import com.goldencode.p2j.security.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.p2j.util.logging.*;

    import javax.servlet.http.*;
    import java.util.*;
    import java.util.concurrent.*;
    import java.util.concurrent.atomic.*;
    import java.util.function.*;
    import java.util.logging.*;
    import java.util.stream.*;

    /** Manager for appserver with multi-session agents. */
    public class MultiSessionAppserverManager
    {
       /** Initialization on demand idiom singleton holder */
       private static class SingletonHolder
       {
          /** Singleton instance */
          private static final MultiSessionAppserverManager INSTANCE = new MultiSessionAppserverManager();
       }

       /** Context-local global data area for MSA session state */
       private static final ContextLocal<WorkArea> local = new ContextLocal<WorkArea>()
       {
          /**
           * Get the weight of this context-local var.
           *
           * @return   The weight of this context-local var.
           */
          @Override
          public WeightFactor getWeight()
          {
             return WeightFactor.LAST;
          }

          /**
           * Initializes the work area, the first time it is requested within a
           * new context.
           *
           * @return   The newly instantiated work area.
           */
          @Override
          protected synchronized WorkArea initialValue()
          {
             return new WorkArea();
          }
       };

       /** Decimal representation of digit characters is in range [48:57], for uppercase latin letters [65:90]. */
       private static final int[] CONNECTION_ID_CHARS = IntStream.concat(IntStream.rangeClosed(48, 57),
                                                                         IntStream.rangeClosed(65, 90))
                                                                 .toArray();

       /** The number of random characters in the connection id. */
       private static final int CONNECTION_ID_RANDOM_CHAR_COUNT = 44;

       /** Thread local flag to indicate if it's a multi-session agent context */
       private static final ThreadLocal<Boolean> isMsaContext = ThreadLocal.withInitial(() -> false);

       /** Logger. */
       private static final CentralLogger LOG = CentralLogger.get(MultiSessionAppserverManager.class);

       /** No connections error msg. */
       private static final String ERR_NO_CONNECTIONS = "No connections available to process request.";

       /** Value to disable the client driven session execution timeout. */
       private static final int DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT = -1;

       /** Pairs of instantiated appserver name : instance. */
       private final Map<String, MultiSessionAppserver> startedAppservers = new ConcurrentHashMap<>();

       /** Pairs of connection id : connection object. */
       private final Map<String, MultiSessionClientConnection> connections = new ConcurrentHashMap<>();

       /** Instance of the scheduled executor where timeout jobs are running for the requests. */
       private final ScheduledExecutorService TIMEOUT_EXECUTOR =
          Executors.newScheduledThreadPool(1, new ThreadFactory()
       {
          /** Thread-safe counter for the thread ids used to run the timeout jobs. */
          private final AtomicInteger threadIdCounter = new AtomicInteger(0);

          /**
           * Constructs a new {@code Thread}.
           *
           * @param    runnable
           *           A runnable to be executed by new thread instance.
           *
           * @return   Constructed thread.
           */
          @Override
          public Thread newThread(Runnable runnable)
          {
             Thread newThread = new Thread(runnable,
                                           "MSA Timeout Executor thread " + threadIdCounter.getAndIncrement());
             newThread.setDaemon(true);
             return newThread;
          }
       });

       /**
        * Consumer anonymous instance accepting connection id and program handle and associating the persistent
        * resource with the connection.
        */
       private final BiConsumer<String, handle> ADD_PROC_FUNC = (connectionId, extProg) ->
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection != null)
          {
             connection.addPersistentProc(extProg);
          }
       };

       /**
        * Consumer anonymous instance accepting connection id and program res id and removing the associating of
        * the persistent resource from the connection.
        */
       private final BiConsumer<String, String> REMOVE_PROC_FUNC = (connectionId, procId) ->
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection != null)
          {
             connection.deletePersistentProc(procId);

             // TODO:
             /*if (connection.isBound())
             {
                msaAppserver.unbind(connection, sessionId, true);

                CoreAppserver.invokeDeactivate(msaDef.getSessionDeactivateProc(),
                                               isConnectionBound(),
                                               connectionId,
                                               SET_RUNNING_PROC_FUNC);
             }*/
          }
       };

       /**
        * Consumer anonymous instance accepting connection id and disconnecting the client connection from the
        * appserver.
        */
       private final Consumer<String> DISCONNECT_FUNC = (connId) -> AppServerManager.disconnect(null, connId);

       /**
        * Consumer anonymous instance accepting connection id and session id and binding the MSA session to the
        * connection.
        */
       private final BiConsumer<String, Integer> BIND_PERSISTENT_FUNC = (connectionId, sessionId) ->
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection == null || connection.isDisconnecting())
          {
             return;
          }
          MultiSessionAppserver msaAppserver = startedAppservers.get(connection.getAppserverName());
          if (msaAppserver == null)
          {
             return;
          }
          msaAppserver.bind(connection, sessionId, true);
       };

       /**
        * Consumer anonymous instance accepting connection id and session id and unbinding the MSA session from
        * the connection.
        */
       private final BiConsumer<String, Integer> UNBIND_PERSISTENT_FUNC = (connectionId, sessionId) ->
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection == null || connection.isDisconnecting())
          {
             return;
          }
          MultiSessionAppserver msaAppserver = startedAppservers.get(connection.getAppserverName());
          if (msaAppserver == null)
          {
             return;
          }
          msaAppserver.unbind(connection, sessionId, true);
       };

       /**
        * Consumer anonymous instance accepting connection id and the name of the currently running program and
        * creating association between them.
        */
       private final BiConsumer<String, String> SET_RUNNING_PROC_FUNC = (connectionId, procName) ->
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection != null)
          {
             connection.setCurrentlyRunningProgram(procName);
          }
       };

       /** The max number of timed out client connections to be kept in the cache. */
       private static final int MAX_CACHED_TIMED_OUT_CONNECTIONS = 100;

       /** Collection of ids of times out client connections */
       private final Map<String, MultiSessionClientConnection> timedOutConnections =
          new LinkedHashMap<String, MultiSessionClientConnection>(MAX_CACHED_TIMED_OUT_CONNECTIONS * 4/3,
                                                                  0.75f,
                                                                  false)
          {
             @Override
             protected boolean removeEldestEntry(Map.Entry<String, MultiSessionClientConnection> eldest)
             {
                return size() > MAX_CACHED_TIMED_OUT_CONNECTIONS;
             }
          };

       /**
        * Consumer anonymous instance accepting a collection of client connections that need to be disconnected
        * due to being timed out.
        */
       private final Consumer<Collection<MultiSessionClientConnection>> REMOVE_TIMEDOUT_CONN_FUNC = clientConnections ->
       {
          boolean isInfoLevelLoggable = LOG.isLoggable(Level.INFO);
          for (MultiSessionClientConnection connection : clientConnections)
          {
             String id = connection.getId();
             if (isInfoLevelLoggable)
             {
                LOG.log(Level.INFO, "Client connection with id %s timed out and will be disconnected.", id);
             }

             synchronized (timedOutConnections)
             {
                timedOutConnections.put(id, connection);
             }

             disconnect(id, true);
          }
       };

       /**
        * Static method for access to the singleton instance of the manager
        *
        * @return  The singleton instance of the manager
        */
       public static MultiSessionAppserverManager getInstance()
       {
          return SingletonHolder.INSTANCE;
       }

       /**
        * Returns the number of MSA agents running for the specified appserver.
        *
        * @param   appserverName
        *          The appserver name.
        *
        * @return  See above.
        */
       public int getNumberOfAgents(String appserverName)
       {
          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          return msaAppserver == null ? 0 : msaAppserver.getNumberOfAgents();
       }

       /**
        * Returns the number of MSA sessions running for the specified appserver.
        *
        * @param   appserverName
        *          The appserver name.
        *
        * @return  See above.
        */
       public int getNumberOfSessions(String appserverName)
       {
          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          return msaAppserver == null ? 0 : msaAppserver.getNumberOfSessions();
       }

       /**
        * Returns <code>true</code> if the current security context is for MSA session.
        *
        * @return  See above.
        */
       public boolean isMsaContext()
       {
          return isMsaContext.get();
       }

       /**
        * Returns <code>true</code> if the specified connection id is for MSA.
        *
        * @param   connectionId
        *          The connection id.
        *
        * @return  See above.
        */
       public boolean isMsaConnection(String connectionId)
       {
          return connectionId.indexOf(".") == CONNECTION_ID_RANDOM_CHAR_COUNT;
       }

       /**
        * Returns the MSA client connection with the specified id.
        *
        * @param   connectionId
        *          The connection id.
        *
        * @return  See above.
        */
       public MultiSessionClientConnection getConnection(String connectionId)
       {
          return connections.get(connectionId);
       }

       /**
        * Returns <code>true</code> if the current client connection is running in Session-Free.
        *
        * @return  See above.
        */
       public boolean isSessionFree()
       {
          return local.get().connection.isSessionFree();
       }

       /**
        * Returns the appserver name executing the currently running request.
        *
        * @return  See above.
        */
       public String getAppserverName()
       {
          MultiSessionClientConnection connection = local.get().connection;
          return connection == null ? null : connection.getAppserverName();
       }

       /**
        * Returns the connection id for the currently running request.
        *
        * @return  See above.
        */
       public String getConnectionId()
       {
          MultiSessionClientConnection connection = local.get().connection;
          return connection == null ? null : connection.getId();
       }

       /**
        * Returns <code>true</code> if the current client connection is bound to an MSA session.
        *
        * @return  See above.
        */
       public boolean isConnectionBound()
       {
          return local.get().connection.isBound();
       }

       /**
        * Returns the MSA session id used for the currently running request.
        *
        * @return  See above.
        */
       public Integer getSessionId()
       {
          return local.get().sessionId;
       }

       /**
        * Returns the agent id used for the currently running request.
        *
        * @return  See above.
        */
       public Integer getAgentId()
       {
          return local.get().agentId;
       }

       /**
        * Returns the client connection of the currently running request.
        *
        * @return  See above.
        */
       public MultiSessionClientConnection getConnection()
       {
          return local.get().connection;
       }

       /**
        * Returns <code>true</code> if the currently running task is initial setup for the session.
        *
        * @return  See above.
        */
       public boolean isInitialSetup()
       {
          return local.get().isStartupProc;
       }

       /**
        * Returns the stacktrace of the original thread of the MsaTask running on the context. Used for debugging.
        *
        * @return  See above.
        */
       public List<StackTraceElement[]> getDebugStackTrace()
       {
          return local.get().debugStackTrace;
       }

       /**
        * Returns the persistent procedure name resolved by its resource id on the connection specified by the
        * connection id.
        *
        * @param   connectionId
        *          The connection id.
        * @param   remoteProcResId
        *          The remote procedure resource id.
        * @param   agentId
        *          The agent id.
        *
        * @return  See above.
        */
       public String getProcedureName(String connectionId, String remoteProcResId, int agentId)
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          return connection == null ? null : connection.getPersistentProcName(remoteProcResId);
       }

       /**
        * Get the SERVER-CONNECTION-CONTEXT attribute of this session.
        *
        * @return   See above.
        */
       public static String getServerConnectionContext()
       {
          return local.get().connection.getServerConnectionContext();
       }

       /**
        * Set the SERVER-CONNECTION-CONTEXT attribute of this session.
        *
        * @param    value
        *           The attribute's value.
        */
       public static void setServerConnectionContext(String value)
       {
          local.get().connection.setServerConnectionContext(value);
       }

       /**
        * Method that specifies the remote procedures the client can execute from the AppServer
        * session.
        *
        * @param    procList
        *           The list of procedure  and name-patterns separated by comma.
        *
        * @return   <code>true</code> if the method succeeded.
        */
       public logical export(character procList)
       {
          MultiSessionClientConnection currentConnection = local.get().connection;
          if (currentConnection == null)
          {
             return new logical(false);
          }
          MultiSessionAppserver currentAppserver = startedAppservers.get(currentConnection.getAppserverName());
          if (currentAppserver == null)
          {
             return new logical(false);
          }
          return currentAppserver.setExports(procList);
       }

       /**
        * Collects and returns the info for the appserver agent.
        *
        * @param    appserverName
        *           The appserver name to trim.
        *
        * @return   The info for the MSA agent.
        */
       public MultiSessionAgentInfo getAgentInfo(String appserverName)
       {
          if (!startedAppservers.containsKey(appserverName))
          {
             return null;
          }
          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          return new MultiSessionAgentInfo(msaAppserver.getStartupTime(),
                                           MultiSessionAgent.SINGLE_AGENT_ID,
                                           msaAppserver.getSessionDefs());
       }

       /**
        * Check if current running appserver knows the given procedure.
        *
        * @param    referent
        *           A persistent procedure referent.
        *
        * @return   <code>true</code> if the persistent procedure is known to the current running appserver.
        */
       public boolean hasProcedure(Object referent)
       {
          MultiSessionClientConnection currentConnection = local.get().connection;
          if (currentConnection == null)
          {
             return false;
          }
          return currentConnection.hasPersistentProc(referent);
       }

       /**
        * Returns <code>true</code> if the appserver specified by the name starts successfully, or is already
        * running.
        *
        * @param   appserverName
        *          The appserver name.
        *
        * @return  See above.
        */
       public boolean start(String appserverName)
       {
          if (startedAppservers.containsKey(appserverName))
          {
             return true;
          }

          MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);

          if (!msaDef.isAllServerSide())
          {
             LOG.severe("MSA appserver can't start without server-side-resources config of 'all'.");
             return false;
          }

          long startNano = System.nanoTime();

          // TODO: configured by ?
          int retryAttempts = 3;

          Throwable appserverStartException = null;
          boolean appserverStartSuccess = false;
          try
          {
             FutureTask<Boolean> agentStartupTask = new FutureTask<>(new Callable<Boolean>()
             {
                private Boolean startAppserver(int retryAttempts)
                throws Exception
                {
                   try
                   {
                      MultiSessionAppserver msaAppserver = new MultiSessionAppserver(msaDef,
                                                                                       connections::values,
                                                                                       REMOVE_TIMEDOUT_CONN_FUNC);
                      startedAppservers.put(appserverName, msaAppserver);
                      connections.put(appserverName, msaAppserver.getAdminConnection());
                      return true;
                   }
                   catch (Throwable t)
                   {
                      if (retryAttempts > 0)
                      {
                         return startAppserver(retryAttempts - 1);
                      }
                      throw t;
                   }
                }

                @Override
                public Boolean call()
                throws Exception
                {
                   return startAppserver(retryAttempts);
                }
             });

             agentStartupTask.run();
             appserverStartSuccess = agentStartupTask.get(msaDef.getAgentListenerTimeout(), TimeUnit.MILLISECONDS);
          }
          catch (Exception e)
          {
             appserverStartException = e.getCause();
          }

          long endMs = (System.nanoTime() - startNano) / 1000000;

          if (appserverStartSuccess)
          {
             LOG.log(Level.INFO, "Agent for appserver '%s' was started successfully (%dms)!", appserverName, endMs);
          }
          else
          {
             startedAppservers.remove(appserverName);

             LOG.log(Level.SEVERE,
                     appserverStartException,
                     "Agent for appserver '%s' could not be started (%dms)!",
                     appserverName,
                     endMs);
          }

          return false;
       }

       /**
        * Restart all agents in the specified appserver.
        *
        * @param    appserverName
        *           The appserver name to restart.
        */
       public void restart(String appserverName)
       {
          if (startedAppservers.containsKey(appserverName))
          {
             MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
             msaAppserver.terminateAppserver(MultiSessionAgent.ExitCode.EXPECTED_4020);
             clearApp(appserverName);
          }

          start(appserverName);
       }

       /**
        * Terminates the selected sessions of the appserver and returns the refreshed list of appserver agents.
        *
        * @param    appserverName
        *           The appserver name
        * @param    sessions
        *           The list of sessions to be terminated
        */
       public void terminateSessions(String appserverName, MultiSessionAgentSessionDef[] sessions)
       {
          if (!startedAppservers.containsKey(appserverName))
          {
             return;
          }

          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          Arrays.stream(sessions)
                .parallel()
                .forEach(sessionDef ->
                            msaAppserver.terminateSession(sessionDef.getAgentId(),
                                                            sessionDef.getSessionId(),
                                                            true,
                                                            false));
       }

       /**
        * Terminates the array of agents in the specified appserver.
        *
        * @param    appserverName
        *           The appserver name to restart.
        * @param    agentIds
        *           The array of agent ids to be terminated.
        */
       public void terminateAgents(String appserverName, int... agentIds)
       {
          if (!startedAppservers.containsKey(appserverName))
          {
             return;
          }
          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          for (int agentId : agentIds)
          {
             LOG.info("Terminating MSA agent with id " + agentId + " from app " + appserverName);
             msaAppserver.terminateAgent(agentId, MultiSessionAgent.ExitCode.EXPECTED_4020);
          }
       }

       /**
        * Trim the agents in the specified appserver.
        *
        * @param    appserverName
        *           The appserver name to trim.
        */
       public void trim(String appserverName)
       {
          if (startedAppservers.containsKey(appserverName))
          {
             MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
             msaAppserver.removeIdleSessions(MultiSessionAgent.SINGLE_AGENT_ID);
          }
       }

       /**
        * Returns {@link AppServerInvocationResult} after initializing the specified proxy procedure, by
        * calling the <code>execute</code> method.
        *
        * @param   hProc
        *          The procedure handle.
        * @param   result
        *          The invocation result container.
        * @param   connectionId
        *          The connection id.
        * @param   externalProcName
        *          The name of the remote external procedure.
        */
       public void obtainProxy(handle hProc,
                               AppServerInvocationResult result,
                               String connectionId,
                               String externalProcName)
       {
          // this request id is needed in the consecutive proxy calls
          int requestId = AppServerManager.uniqueId();

          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.OBTAIN_PROXY,
                                                             connectionId,
                                                             requestId,
                                                             DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT,
                                                             true);

          InvokeConfig invokeConfig = new InvokeConfig();
          invokeConfig.setProcedureHandle(hProc);
          if (!standardInvoke.determineConnection(invokeConfig, externalProcName))
          {
             return;
          }

          standardInvoke.exec(() -> {
             CoreAppserver.obtainProxy(hProc,
                                       result,
                                       standardInvoke.msaAppserver.getExports(),
                                       standardInvoke.msaDef.getSessionActivateProc(),
                                       standardInvoke.msaDef.getSessionDeactivateProc(),
                                       standardInvoke.invokeConnection::isBound,
                                       standardInvoke.invokeConnection.getId(),
                                       standardInvoke.agentId,
                                       standardInvoke.sessionId,
                                       externalProcName,
                                       SET_RUNNING_PROC_FUNC,
                                       BIND_PERSISTENT_FUNC,
                                       ADD_PROC_FUNC,
                                       DISCONNECT_FUNC);
          });
       }

       /**
        * Returns {@link AppServerInvocationResult} after initializing the specified proxy procedure, by
        * calling the <code>execute</code> method.
        *
        * @param   connectionId
        *          The connection id.
        * @param   requestId
        *          The request id.
        * @param   externalProcResId
        *          The external procedure resource id.
        * @param   agentId
        *          The agent id where this remote procedure was created.
        * @param   transactionDistinct
        *          Flag indicating if the TRANSACTION DISTINCT clause is in effect.
        * @param   modes
        *          A string representation of the modes of each parameter. May be <code>null</code>
        * @param   args
        *          The procedure's arguments.
        *
        * @return  See above.
        */
       public AppServerInvocationResult initializeProxy(String connectionId,
                                                        int requestId,
                                                        String externalProcResId,
                                                        int agentId,
                                                        boolean transactionDistinct,
                                                        String modes,
                                                        Object[] args)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INIT_PROXY,
                                                             connectionId,
                                                             requestId,
                                                             DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT,
                                                             true);

          if (!standardInvoke.determineConnection(externalProcResId))
          {
             return standardInvoke.result;
          }

          return standardInvoke.exec(() -> {
             ExternalProgramWrapper extProg = standardInvoke.invokeConnection.getProcWrapper(externalProcResId);

             CoreAppserver.initializeProxy(standardInvoke.result,
                                           externalProcResId,
                                           extProg,
                                           standardInvoke.invokeConnection.getId(),
                                           agentId,
                                           getSessionId(),
                                           requestId,
                                           transactionDistinct,
                                           modes,
                                           args,
                                           SET_RUNNING_PROC_FUNC,
                                           REMOVE_PROC_FUNC,
                                           UNBIND_PERSISTENT_FUNC,
                                           standardInvoke.invokeConnection::isBound,
                                           standardInvoke.msaDef.getSessionDeactivateProc());
          });
       }

       /**
        * Delete the remote procedure, specified by the resource id. This must be called at the same
        * time when the local proxy procedure is deleted.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    procResId
        *           The ID of a remote procedure which needs to be deleted.
        * @param    agentId
        *           The agent ID where this remote procedure was created.
        *
        * @return   <code>true</code> if the remote procedure was deleted.
        */
       public boolean deleteProcedure(String connectionId, String procResId, int agentId)
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection == null || connection.isDisconnecting())
          {
             return false;
          }

          String appserverName = connection.getAppserverName();
          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          if (msaAppserver == null)
          {
             return false;
          }

          MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
          MultiSessionClientConnection invokeConnection = connection.findConnectionFor(procResId);

          CancellableCompositeTask<AppServerInvocationResult> invokeTask =
             new CancellableCompositeTask<>(new MsaCancellableCallable()
             {
                @Override
                public AppServerInvocationResult call()
                throws Exception
                {
                   int sessionId = reserveSession(msaAppserver, invokeConnection);

                   msaAppserver.runInTopScope(invokeConnection, sessionId, () ->
                   {
                      invokeConnection.deletePersistentProc(procResId);

                      if (invokeConnection.isBound())
                      {
                         if (!invokeConnection.hasPersistentProc())
                         {
                            msaAppserver.unbind(invokeConnection, sessionId, true);
                         }

                         CoreAppserver.invokeDeactivate(msaDef.getSessionDeactivateProc(),
                                                        isConnectionBound(),
                                                        invokeConnection.getId(),
                                                        SET_RUNNING_PROC_FUNC);
                      }
                   });
                   return null;
                }
             }, MultiSessionTaskType.DELETE);

          AppServerInvocationResult result = new AppServerInvocationResult();
          handleExec(invokeTask,
                     result,
                     invokeConnection,
                     msaDef.getRequestWaitTimeout(),
                     msaDef.getSessionExecutionTimeLimit());

          return result.getError() != null;
       }

       /**
        * Delete the remote object, specified by the resource ID. This must be called at the same
        * time when the local proxy object is deleted.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    objResId
        *           The ID of a remote object which needs to be deleted.
        * @param    agentId
        *           The agent ID where this remote procedure was created.
        *
        * @return   <code>true</code> if the remote object was deleted.
        */
       public boolean deleteObject(String connectionId, String objResId, int agentId)
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection == null || connection.isDisconnecting())
          {
             return false;
          }

          String appserverName = connection.getAppserverName();
          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          if (msaAppserver == null)
          {
             return false;
          }

          MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
          MultiSessionClientConnection invokeConnection = connection.findConnectionFor(objResId);

          CancellableCompositeTask<AppServerInvocationResult> invokeTask =
             new CancellableCompositeTask<>(new MsaCancellableCallable()
             {
                @Override
                public AppServerInvocationResult call()
                throws Exception
                {
                   int sessionId = reserveSession(msaAppserver, invokeConnection);

                   msaAppserver.runInTopScope(invokeConnection, sessionId, () ->
                   {
                      invokeConnection.deletePersistentObject(objResId);

                      if (invokeConnection.isBound())
                      {
                         if (!invokeConnection.hasPersistentProc())
                         {
                            msaAppserver.unbind(invokeConnection, sessionId, true);
                         }

                         CoreAppserver.invokeDeactivate(msaDef.getSessionDeactivateProc(),
                                                        invokeConnection.isBound(),
                                                        invokeConnection.getId(),
                                                        SET_RUNNING_PROC_FUNC);
                      }
                   });
                   return null;
                }
             }, MultiSessionTaskType.DELETE);

          AppServerInvocationResult result = new AppServerInvocationResult();
          handleExec(invokeTask,
                     result,
                     invokeConnection,
                     msaDef.getRequestWaitTimeout(),
                     msaDef.getSessionExecutionTimeLimit());

          return result.getError() != null;
       }

       /**
        * Send a STOP condition to all the requests on the specified connection and clear the queue.
        *
        * @param    connectionId
        *           The appserver connection id.
        */
       public void sendStop(String connectionId)
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection == null || connection.isDisconnecting())
          {
             return;
          }

          connection.cancelTasks();
       }

       /**
        * Establish a new appserver connection. This will invoke any connect procedure for this
        * appserver, to which the given <code>user</code>, <code>pwd</code> and <code>serverInfo</code>
        * parameters will be passed.
        * <p>
        * The appserver connection must be configured to start under the given P2J-level subject ID.
        *
        * @param    appserverName
        *           The name of the target appserver.
        * @param    sessionFree
        *           Flag indicating if the connection request is for a session-free operating mode.
        * @param    user
        *           The first parameter to pass to the connect procedure, if any.
        * @param    pwd
        *           The second parameter to pass to the connect procedure, if any.
        * @param    serverInfo
        *           The third parameter to pass to the connect procedure, if any.
        * @param    requestInfo
        *           The remote {@link com.goldencode.p2j.oo.lang.OerequestInfo} details, as set by
        *           {@link com.goldencode.p2j.oo.lang.OerequestInfo#toArray()}.
        * @param    webServiceType
        *           The type of web service the worker is used for.
        *
        * @return   An {@link AppServerInvocationResult} instance with any result or caught errors.
        */
       public AppServerInvocationResult connect(String appserverName,
                                                boolean sessionFree,
                                                String user,
                                                String pwd,
                                                String serverInfo,
                                                Object[] requestInfo,
                                                String webServiceType)
       {
          AppServerInvocationResult result = new AppServerInvocationResult();

          if (!startedAppservers.containsKey(appserverName))
          {
             result.setError(new NumberedException("Application server connect failure.", 5468));
             return result;
          }

          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);

          if (!msaAppserver.isRunning())
          {
             result.setError(new NumberedException("Application server connect failure.", 5468));
             return result;
          }

          MultiSessionClientConnection connection = createNewClientConnection(appserverName,
                                                                              null,
                                                                              sessionFree,
                                                                              webServiceType != null);
          String connectionId = connection.getId();

          MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);

          CancellableCompositeTask<AppServerInvocationResult> connectTask =
             new CancellableCompositeTask<>(new MsaCancellableCallable()
             {
                @Override
                public AppServerInvocationResult call()
                throws Exception
                {
                   if (!sessionFree)
                   {
                      msaAppserver.log(
                         "Application Server connected with connection id: " + connectionId + ". (8358)",
                         LoggingLevel.BASIC,
                         LoggingEntryType.DB_Connects,
                         LoggingExecEnv.AS_ADMIN);
                   }

                   String connectProc = msaDef.getSessionConnectProc();
                   if (sessionFree || connectProc == null)
                   {
                      result.setConnectionID(connectionId);
                      result.setConnectionModel(AppServerSessionModel.SESSION_FREE);
                      return result;
                   }

                   int sessionId = reserveSession(msaAppserver, connection);

                   int threadId = (int) Thread.currentThread().getId();

                   result.setAgentId(MultiSessionAgent.SINGLE_AGENT_ID);
                   result.setConnectionID(connectionId);
                   result.setConnectionModel(AppServerSessionModel.SESSION_MANAGED);
                   result.setSessionId(sessionId);
                   result.setThreadId(threadId);

                   if (isCancelled() || sessionId == MultiSessionAppserver.INVALID_SESSION_ID)
                   {
                      String errMsg = getCancelCause() == CancelCause.SEND_STOP ? "" : ERR_NO_CONNECTIONS;
                      result.setError(new ErrorConditionException(errMsg));
                      return result;
                   }

                   SET_RUNNING_PROC_FUNC.accept(connectionId, connectProc);

                   msaAppserver.runInTopScope(result, requestInfo, connection, sessionId, connectProc, () ->
                   {
                      ControlFlowOps.invokeExternalProcedure(new character(connectProc),
                                                             false,
                                                             null,
                                                             false,
                                                             null,
                                                             "III",
                                                             new character(user),
                                                             new character(pwd),
                                                             new character(serverInfo));
                   }, getIsRunningSupplier(), getCancelledSupplier());

                   SET_RUNNING_PROC_FUNC.accept(connectionId, null);

                   return result;
                }
             }, MultiSessionTaskType.CONNECT);

          return handleExec(connectTask,
                            result,
                            connection,
                            msaDef.getRequestWaitTimeout(),
                            msaDef.getSessionExecutionTimeLimit());
       }

       /**
        * Disconnect and cleanup the appserver connection.
        *
        * @param    connectionId
        *           The connection id.
        * @param    shutdownBoundSession
        *           Flag to indicate the bound MSA session should be shut down.
        *
        * @return   <code>true</code> if an active connection has been disconnected.
        */
       public boolean disconnect(String connectionId, boolean shutdownBoundSession)
       {
          MultiSessionClientConnection connection = connections.get(connectionId);
          if (connection == null || connection.isDisconnecting())
          {
             return false;
          }

          connection.setDisconnecting(true);
          connection.disableLegacyErrors(false);

          String appserverName = connection.getAppserverName();
          if (!startedAppservers.containsKey(appserverName))
          {
             connections.remove(connectionId);
             return false;
          }

          MultiSessionAppserver msaAppserver = startedAppservers.get(appserverName);
          if (!msaAppserver.isRunning())
          {
             connections.remove(connectionId);
             return false;
          }

          MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appserverName);
          String disconnectProc = msaDef.getSessionDisconnProc();

          int sessionId = MultiSessionAppserver.INVALID_SESSION_ID;
          try
          {
             sessionId = msaAppserver.reserveSession(connection).get();
          }
          catch (Exception e)
          {
             LOG.info("Reserving MSA session for disconnect failed.", e);
          }

          // TODO: OE waits for the currently running async proc and then disconnects, clearing next in queue

          if (sessionId == MultiSessionAppserver.INVALID_SESSION_ID)
          {
             LOG.warning(ERR_NO_CONNECTIONS);
             clearConnections(msaAppserver, connection, shutdownBoundSession);
             return false;
          }

          final int finalSessionId = sessionId;
          try
          {
             msaAppserver.runInTopScope(null, null, connection, sessionId, disconnectProc, () ->
             {
                SET_RUNNING_PROC_FUNC.accept(connectionId, disconnectProc);

                if (!connection.isSessionFree() && disconnectProc != null)
                {
                   ControlFlowOps.invokeExternalProcedure(new character(disconnectProc),
                                                          false,
                                                          null,
                                                          false,
                                                          null,
                                                          null);
                }
                SET_RUNNING_PROC_FUNC.accept(connectionId, null);

                for (MultiSessionClientConnection childConnection : connection.getChildConnections())
                {
                   Integer boundSessionId = childConnection.getBoundSessionId();
                   boolean doesContextExist =
                      MultiSessionAppserverSecurityManager.getInstance()
                                                          .hasContext(appserverName, boundSessionId);
                   if (boundSessionId == null || finalSessionId == boundSessionId || !doesContextExist)
                   {
                      msaAppserver.deleteConnection(childConnection, false);
                   }
                   else
                   {
                      try
                      {
                         Runnable runnable = () -> {
                            msaAppserver.deleteConnection(childConnection, shutdownBoundSession);
                         };
                         msaAppserver.runInSessionContext(connection,
                                                          boundSessionId,
                                                          null,
                                                          runnable,
                                                          null,
                                                          null,
                                                          null);
                      }
                      catch (Exception e)
                      {
                         if (LOG.isLoggable(Level.FINE))
                         {
                            LOG.log(Level.FINE, e, "Disconnect error: ");
                         }
                      }
                   }
                }

                msaAppserver.deleteConnection(connection, shutdownBoundSession);
             }, null, null);
          }
          catch (Exception e)
          {
             if (LOG.isLoggable(Level.FINE))
             {
                LOG.log(Level.FINE, e, "Disconnect error: ");
             }
          }

          clearConnections(msaAppserver, connection, shutdownBoundSession);

          return true;
       }

       /**
        * Invoke a remote external procedure, internal procedure or user-defined function. The
        * validation for the program name and arguments will be done on the remote side.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    cfg
        *           The invoke configuration.
        *
        * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
        *           arguments and any returned value.
        */
       public AppServerInvocationResult invoke(String connectionId, int requestId, InvokeConfig cfg)
       {
          return invoke(connectionId, requestId, cfg, null, -1);
       }

       /**
        * Invoke a remote external procedure, internal procedure or user-defined function. The
        * validation for the program name and arguments will be done on the remote side.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    cfg
        *           The invoke configuration.
        * @param    inProcResId
        *           In handle resource id.
        * @param    execTimeout
        *           The maximum allowed time for this invocation to complete. In milliseconds.
        *           Use <code>0</code> to disable the timeout.
        *
        * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
        *           arguments and any returned value.
        */
       public AppServerInvocationResult invoke(String connectionId,
                                               int requestId,
                                               InvokeConfig cfg,
                                               String inProcResId,
                                               long execTimeout)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                             connectionId, requestId, execTimeout, false);

          if (!standardInvoke.determineConnection(cfg, inProcResId))
          {
             return standardInvoke.result;
          }

          return standardInvoke.exec(() -> {
             if (standardInvoke.isPersistentProcRun)
             {
                CoreAppserver.runPersistent(standardInvoke.result,
                                            true,
                                            standardInvoke.msaAppserver.getExports(),
                                            standardInvoke.msaDef.getSessionActivateProc(),
                                            standardInvoke.msaDef.getSessionDeactivateProc(),
                                            standardInvoke.invokeConnection::isBound,
                                            standardInvoke.invokeConnection.getId(),
                                            MultiSessionAgent.SINGLE_AGENT_ID,
                                            standardInvoke.threadId,
                                            standardInvoke.sessionId,
                                            cfg,
                                            SET_RUNNING_PROC_FUNC,
                                            BIND_PERSISTENT_FUNC,
                                            UNBIND_PERSISTENT_FUNC,
                                            ADD_PROC_FUNC,
                                            DISCONNECT_FUNC);
             }
             else
             {
                CoreAppserver.invoke(standardInvoke.result,
                                     true,
                                     true,
                                     standardInvoke.msaAppserver.getExports(),
                                     standardInvoke.msaDef.getSessionActivateProc(),
                                     standardInvoke.msaDef.getSessionDeactivateProc(),
                                     standardInvoke.invokeConnection::isBound,
                                     standardInvoke.invokeConnection.getId(),
                                     MultiSessionAgent.SINGLE_AGENT_ID,
                                     standardInvoke.threadId,
                                     standardInvoke.sessionId,
                                     cfg,
                                     inProcResId,
                                     SET_RUNNING_PROC_FUNC,
                                     REMOVE_PROC_FUNC,
                                     DISCONNECT_FUNC);
             }
          });
       }

       /**
        * Perform a pseudo-dynamic call where the target procedure doesn't receive the parameter as arguments at
        * the Java method definition, but instead they are managed via {@link LegacyOpenClientCaller} APIs.
        * <p>
        * This allows the target program to act as a controller, where it can prepare the arguments, perform
        * security checks, etc, before dispatching the call to the real target, which can be resolved from the
        * arguments or in some other way.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    execTimeout
        *           The maximum allowed time for this invocation to complete.  In milliseconds.
        *           Use <code>0</code> to disable the timeout.
        * @param    procedure
        *           The target external program.  Must have no parameters defined.
        * @param    argTypes
        *           The argument types, one of the e.g. {@link LegacyJavaAppserverApi#integer()} values.
        * @param    modes
        *           The argument modes, one of the (I)NPUT, (O)UTPUT or INP(U)T-OUTPUT values.
        * @param    args
        *           The actual arguments.
        *
        * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT arguments
        *           and any returned value.
        */
       public AppServerInvocationResult invokeWithArgs(String connectionId,
                                                       int requestId,
                                                       long execTimeout,
                                                       String procedure,
                                                       int[] argTypes,
                                                       String modes,
                                                       Object[] args)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                             connectionId, requestId, execTimeout, false);

          return standardInvoke.exec(() -> {
             CoreAppserver.invokeWithArgs(standardInvoke.result,
                                          standardInvoke.invokeConnection.getId(),
                                          standardInvoke.msaDef.getSessionActivateProc(),
                                          standardInvoke.msaDef.getSessionDeactivateProc(),
                                          standardInvoke.invokeConnection::isBound,
                                          SET_RUNNING_PROC_FUNC,
                                          DISCONNECT_FUNC,
                                          procedure,
                                          argTypes,
                                          modes,
                                          args);
          });
       }

       /**
        * Create a new instance of the specified qualified class name.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    execTimeout
        *           The maximum allowed time for this invocation to complete.  In milliseconds.
        *           Use <code>0</code> to disable the timeout.
        * @param    name
        *           The legacy qualified class name for the legacy class.
        * @param    modes
        *           A string representation of the modes of each parameter. May be <code>null</code>
        * @param    args
        *           The constructor's arguments.
        *
        * @return   A {@link AppServerInvocationResult} instance, which holds the code for the remote
        *           object, all updated OUTPUT/INPUT-OUTPUT arguments and any returned value.
        */
       public AppServerInvocationResult newInstance(String connectionId,
                                                    int requestId,
                                                    long execTimeout,
                                                    character name,
                                                    String modes,
                                                    Object[] args)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.NEW_INSTANCE,
                                                             connectionId, requestId, execTimeout, true);

          if (!standardInvoke.determinePseudoPersistentConnection())
          {
             return standardInvoke.result;
          }

          return standardInvoke.exec(() -> {
             CoreAppserver.newInstance(standardInvoke.result,
                                       standardInvoke.invokeConnection.getId(),
                                       MultiSessionAgent.SINGLE_AGENT_ID,
                                       standardInvoke.sessionId,
                                       standardInvoke.msaDef.getSessionActivateProc(),
                                       standardInvoke.msaDef.getSessionDeactivateProc(),
                                       standardInvoke.invokeConnection::isBound,
                                       SET_RUNNING_PROC_FUNC,
                                       BIND_PERSISTENT_FUNC,
                                       UNBIND_PERSISTENT_FUNC,
                                       ADD_PROC_FUNC,
                                       DISCONNECT_FUNC,
                                       name,
                                       modes,
                                       args);
          });
       }

       /**
        * Invoke a method in a remotely instantiated class.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    execTimeout
        *           The maximum allowed time for this invocation to complete.  In milliseconds.
        *           Use <code>0</code> to disable the timeout.
        * @param    returnValue
        *           Flag indicating if the return value is required.
        * @param    name
        *           The legacy 4GL name for the class method.
        * @param    resId
        *           The code of a remotely instantiated object in which to invoke the method.
        * @param    agentId
        *           The agent ID where this remote procedure was created.
        * @param    modes
        *           A string representation of the modes of each parameter. May be <code>null</code>
        * @param    args
        *           The method's arguments.
        *
        * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
        *           arguments and any returned value.
        */
       public AppServerInvocationResult invokeMethod(String connectionId,
                                                     int requestId,
                                                     long execTimeout,
                                                     boolean returnValue,
                                                     character name,
                                                     String resId,
                                                     int agentId,
                                                     String modes,
                                                     Object[] args)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                             connectionId, requestId, execTimeout, true);

          if (!standardInvoke.determineConnection(resId))
          {
             return standardInvoke.result;
          }

          return standardInvoke.exec(() -> {
             CoreAppserver.invokeMethod(standardInvoke.result,
                                        true,
                                        standardInvoke.invokeConnection.getId(),
                                        standardInvoke.invokeConnection::isBound,
                                        SET_RUNNING_PROC_FUNC,
                                        returnValue,
                                        name,
                                        resId,
                                        modes,
                                        args);
          });
       }

       /**
        * Invoke the given Java method directly. Used by {@link JavaRestService}.
        * <p>
        * All arguments are sent by references: if it was marked as an OUTPUT or INPUT-OUTPUT, any change to that
        * instance will be reflected back to the caller.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    execTimeout
        *           The maximum allowed time for this invocation to complete.  In milliseconds.
        *           Use <code>0</code> to disable the timeout.
        * @param    cfg
        *           The configuration with the Java method to be invoked.
        *
        * @return   A {@link AppServerInvocationResult} instance with all updated OUTPUT/INPUT-OUTPUT
        *           arguments and any returned value.
        */
       public AppServerInvocationResult invokeJavaMethod(String connectionId,
                                                         int requestId,
                                                         int execTimeout,
                                                         JavaInvokeConfig cfg)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.INVOKE,
                                                             connectionId, requestId, execTimeout, false);

          return standardInvoke.exec(() -> {
             CoreAppserver.invokeJavaMethod(standardInvoke.result,
                                            standardInvoke.invokeConnection.getId(),
                                            true,
                                            standardInvoke.msaDef.getSessionActivateProc(),
                                            standardInvoke.msaDef.getSessionDeactivateProc(),
                                            standardInvoke.invokeConnection::isBound,
                                            SET_RUNNING_PROC_FUNC,
                                            DISCONNECT_FUNC,
                                            cfg);
          });
       }

       /**
        * Handle a generic web service request.  This exposes the Jetty request and response objects
        * to the Agent which will process it, so it is required for the appserver to be local.
        *
        * @param    connectionId
        *           The appserver connection id.
        * @param    requestId
        *           The ID of the request.
        * @param    execTimeout
        *           The maximum allowed time for this invocation to complete.  In milliseconds.
        *           Use <code>0</code> to disable the timeout.
        * @param    handler
        *           The {@link com.goldencode.p2j.oo.web.WebHandler} implementation.
        * @param    basepath
        *           The service basepath.
        * @param    paths
        *           The split paths for this handler.
        * @param    token
        *           When not null, it represents the token of a FWD context created for an authenticated and
        *           authorized web service call.  The API call will be performed in that context.
        * @param    target
        *           The request original target.
        * @param    request
        *           The Jetty request.
        * @param    response
        *           The Jetty response.
        *
        * @return   A {@link AppServerInvocationResult} instance with any error thrown during the web service
        *           processing.
        */
       public AppServerInvocationResult handleWebService(String connectionId,
                                                         int requestId,
                                                         long execTimeout,
                                                         Class<? extends WebHandler> handler,
                                                         String basepath,
                                                         String[] paths,
                                                         String token,
                                                         String target,
                                                         HttpServletRequest request,
                                                         HttpServletResponse response)
       {
          StandardInvoke standardInvoke = new StandardInvoke(MultiSessionTaskType.WEB,
                                                             connectionId, requestId, execTimeout, false);

          return standardInvoke.exec(() -> {
             CoreAppserver.handleWebService(standardInvoke.result,
                                            standardInvoke.invokeConnection.getId(),
                                            handler,
                                            basepath,
                                            paths,
                                            target,
                                            request,
                                            response,
                                            SET_RUNNING_PROC_FUNC);
          });
       }

       /**
        * Disable legacy OO errors thrown on this connection.
        *
        * @param    connectionId
        *           The appserver connection id.
        */
       public void disableLegacyErrors(String connectionId)
       {
          MultiSessionClientConnection connection = getConnection(connectionId);
          if (connection != null)
          {
             connection.disableLegacyErrors(true);
          }
       }

       /**
        * Check if the current {@link WorkArea#connection connection} should suppress legacy OO errors.
        *
        * @return   See above.
        */
       public boolean hasLegacyErrorsDisabled()
       {
          MultiSessionClientConnection currentConnection = local.get().connection;
          return currentConnection != null && currentConnection.hasLegacyErrorsDisabled();
       }

       /**
        * Get the SERVER-CONNECTION-BOUND-REQUEST attribute of this session.
        *
        * @return   See above.
        */
       public static boolean isServerConnectionBoundRequest()
       {
          WorkArea wa = local.get();
          if (!isMsaContext.get() || wa.connection == null || wa.connection.isSessionFree())
          {
             return false;
          }

          return wa.connection.getServerConnectionBoundRequest();
       }

       /**
        * Set the SERVER-CONNECTION-BOUND-REQUEST attribute of this session.
        *
        * @param   bind
        *          The bound state.
        */
       public static void setServerConnectionBoundRequest(boolean bind)
       {
          WorkArea wa = local.get();
          if (!isMsaContext.get())
          {
             return;
          }

          String appserverName = wa.connection.getAppserverName();
          MultiSessionAppserver msaAppserver = MultiSessionAppserverManager
             .getInstance().startedAppservers.get(appserverName);
          if (msaAppserver == null)
          {
             return;
          }

          if (wa.isStartupProc)
          {
             LegacyLogOps.logMgr().writeMessage(
                "It is not allowed to set SERVER-CONNECTION-BOUND-REQUEST to 'yes' " +
                   "in the startup  procedure of a stateless server. (8024)",
                LoggingEntryType.ASPlumbing,
                null,
                LoggingSubsys.EMPTY,
                LoggingLevel.ERROR);

             // TODO: replace with proper exception instead of logging separate
             throw new ErrorConditionException(
                "It is not allowed to set SERVER-CONNECTION-BOUND-REQUEST to 'yes' " +
                   "in the startup  procedure of a stateless server.", 8024);
          }

          if (wa.connection.isSessionFree() || wa.connection == null || wa.connection.isDisconnecting())
          {
             return;
          }

          wa.connection.setServerConnectionBoundRequest(bind);

          if (bind)
          {
             msaAppserver.bind(wa.connection, wa.sessionId, false);
          }
          else
          {
             msaAppserver.unbind(wa.connection, wa.sessionId, false);
          }
       }

       /**
        * Sets details of the running task to the current context and prepares the OeRequestInfo.
        *
        * @param    connection
        *           The appserver connection.
        * @param    agentId
        *           The ID of the agent.
        * @param    sessionId
        *           The ID of the session.
        * @param    isInitialSetup
        *           <code>true</code> if currently running agent/session initialization.
        * @param    debugStackTrace
        *           Stacktrace of the original thread of the MsaTask. Used for debugging.
        */
       void setMsaContext(MultiSessionClientConnection connection,
                          Integer agentId,
                          Integer sessionId,
                          boolean isInitialSetup,
                          List<StackTraceElement[]> debugStackTrace)
       {
          isMsaContext.set(true);

          WorkArea wa = local.get();
          wa.connection = connection;
          wa.agentId = agentId;
          wa.sessionId = sessionId;
          wa.isStartupProc = isInitialSetup;
          wa.debugStackTrace = debugStackTrace;

          if (isInitialSetup)
          {
             CoreAppserver.setOeRequestInfo(null, true, null, null, null);
          }
          else
          {
             CoreAppserver.setOeRequestInfo(null,
                                            true,
                                            agentId,
                                            (int) Thread.currentThread().getId(),
                                            sessionId);
          }
       }

       /**
        * Resets the details of the running task in the current context and resets the OeRequestInfo.
        */
       void resetMsaContext()
       {
          WorkArea wa = local.get();
          wa.agentId = null;
          wa.connection = null;
          wa.sessionId = null;
          wa.isStartupProc = false;
          wa.debugStackTrace = null;

          CoreAppserver.setOeRequestInfo(null, true, null, null, null);
       }

       /**
        * Clears references to an appserver from the manager.
        *
        * @param    appserverName
        *           The appserver name.
        */
       void clearApp(String appserverName)
       {
          startedAppservers.remove(appserverName);
          connections.entrySet()
                     .removeIf(entry -> entry.getValue().getAppserverName().equals(appserverName));
       }

       /**
        * Clears references to a connection from the MSA manager and the appserver.
        *
        * @param    msaAppserver
        *           The appserver.
        * @param    primaryConnection
        *           The primary connection to be cleared.
        * @param    shutdownBoundSession
        *           <code>true</code> if the bound MSA session should be shut down.
        */
       private void clearConnections(MultiSessionAppserver msaAppserver,
                                     MultiSessionClientConnection primaryConnection,
                                     boolean shutdownBoundSession)
       {
          for (MultiSessionClientConnection childConnection : primaryConnection.getChildConnections())
          {
             clearConnection(msaAppserver, childConnection.getId(), shutdownBoundSession);
          }

          clearConnection(msaAppserver, primaryConnection.getId(), shutdownBoundSession);
       }

       /**
        * Clears references to a connection from the MSA manager and the appserver.
        *
        * @param    msaAppserver
        *           The appserver.
        * @param    connectionId
        *           The id of the connection to be cleared.
        * @param    shutdownBoundSession
        *           <code>true</code> if the bound MSA session should be shut down.
        */
       private void clearConnection(MultiSessionAppserver msaAppserver,
                                    String connectionId,
                                    boolean shutdownBoundSession)
       {
          connections.remove(connectionId);
          msaAppserver.clearConnection(connectionId, shutdownBoundSession);
       }

       /**
        * Returns a newly created client connection.
        *
        * @param    appserverName
        *           The appserver name.
        * @param    parentConnection
        *           The parent connection if such exists.
        * @param    isSessionFree
        *           <code>true</code> if the connection runs in Session-free session model.
        *
        * @return   See above.
        */
       private MultiSessionClientConnection createNewClientConnection(String appserverName,
                                                                      MultiSessionClientConnection parentConnection,
                                                                      boolean isSessionFree)
       {
          return createNewClientConnection(appserverName, parentConnection, isSessionFree, false);
       }

       /**
        * Returns a newly created client connection.
        *
        * @param    appserverName
        *           The appserver name.
        * @param    parentConnection
        *           The parent connection if such exists.
        * @param    isSessionFree
        *           <code>true</code> if the connection runs in Session-free session model.
        * @param    isForWebService
        *           The connection is used by a web service.
        *
        * @return   See above.
        */
       private MultiSessionClientConnection createNewClientConnection(String appserverName,
                                                                      MultiSessionClientConnection parentConnection,
                                                                      boolean isSessionFree,
                                                                      boolean isForWebService)
       {
          String newConnectionId = createConnectionId(appserverName);
          MultiSessionClientConnection connection = new MultiSessionClientConnection(newConnectionId,
                                                                                     parentConnection,
                                                                                     appserverName,
                                                                                     isSessionFree,
                                                                                     isForWebService);
          connections.put(newConnectionId, connection);
          return connection;
       }

       /**
        * Creates a MSA connection id with 80 random uppercase letters and digits symbols + "." +
        * appserverName.
        *
        * @param    appserverName
        *           The appserver name.
        *
        * @return   The connection id.
        */
       private String createConnectionId(String appserverName)
       {
          Random randomGenerator = new Random();
          StringBuilder sb = new StringBuilder();

          for (int i = 0; i < CONNECTION_ID_RANDOM_CHAR_COUNT; i++)
          {
             int index = randomGenerator.nextInt(CONNECTION_ID_CHARS.length);
             char randomChar = (char) CONNECTION_ID_CHARS[index];
             sb.append(randomChar);
          }

          sb.append(".").append(appserverName);

          return sb.toString();
       }

       /**
        * Generic result handling for MSA tasks.
        *
        * @param    task
        *           The wrapper for the MSA task.
        * @param    result
        *           The instance of the result object to be used.
        * @param    connection
        *           The client connection.
        * @param    requestWaitTimeout
        *           The timeout for cancelling the request before it starts running (the time for finding an
        *           MSA session and a free worker thread).
        * @param    sessionExecutionTimeLimit
        *           The time limit for running the request.
        *
        * @return   A {@link AppServerInvocationResult} instance with any error thrown during the execution.
        */
       private AppServerInvocationResult handleExec(CancellableCompositeTask<AppServerInvocationResult> task,
                                                    AppServerInvocationResult result,
                                                    MultiSessionClientConnection connection,
                                                    int requestWaitTimeout,
                                                    int sessionExecutionTimeLimit)
       {
          connection.addTask(task);
          try
          {
             if (requestWaitTimeout > 0)
             {
                Runnable timeoutRunnable = () ->
                {
                   synchronized (task.getRunningSupplier())
                   {
                      if (!task.getRunningSupplier().get())
                      {
                         task.cancel(true);
                      }
                   }
                };
                TIMEOUT_EXECUTOR.schedule(timeoutRunnable, requestWaitTimeout, TimeUnit.MILLISECONDS);
             }

             task.run();

             return sessionExecutionTimeLimit == 0 ?
                task.get() :
                task.get(sessionExecutionTimeLimit, TimeUnit.SECONDS);
          }
          catch (Exception e)
          {
             if (e instanceof CancellationException)
             {
                String msg = task.getCancelCause() == CancelCause.REQUEST_TIMEOUT ?
                   ERR_NO_CONNECTIONS :
                   task.getCancelCause() == null ? "" : task.getCancelCause().toString();

                if (task.getCancelCause() == CancelCause.REQUEST_TIMEOUT)
                {
                   LOG.warning(msg);
                }

                result.setError(new ErrorConditionException(msg));
             }
             else if (e instanceof TimeoutException)
             {
                // sessionExecutionTimeLimit
                task.sendStop();
                result.setError(new StopConditionException("Session execution timed out."));
             }
             else if (e instanceof ExecutionException)
             {
                Throwable cause = e.getCause();
                // the MsaTask Future throws nested ExecutionException
                result.setError(cause instanceof ExecutionException ? cause.getCause() : cause);
             }
             else
             {
                result.setError(e);
             }
             return result;
          }
          finally
          {
             connection.removeTask(task);
          }
       }

       /**
        * A class to wrap up the standard handling of invoke calls: connection management, creating and executing
        * MSA task wrappers that reserve an MSA session and set up the invoke details in the result.
        */
       private class StandardInvoke
       {
          /** The task type. */
          private final MultiSessionTaskType taskType;

          /** The id of the main connection. */
          private final String mainConnectionId;

          /** The request id. */
          private final int requestId;

          /** The session execution timeout specified by the client. */
          private final long clientDrivenSessionExecutionTimeout;

          /** The result instance. */
          private final AppServerInvocationResult result;

          /** Flag to indicate the wrapper has already validated the connection before starting execution. */
          private final AtomicBoolean isConnectionValidationDone = new AtomicBoolean();

          /** The appserver definition. */
          private MultiSessionAppserverDefinition msaDef;

          /** The appserver itself. */
          private MultiSessionAppserver msaAppserver;

          /** The main connection. */
          private MultiSessionClientConnection mainConnection;

          /** The invoke connection. */
          private MultiSessionClientConnection invokeConnection;

          /** Flag to indicate if the invoke should run in persistent mode. */
          private boolean isPersistentProcRun;

          /** The agent id of the reserved session id. */
          private Integer agentId;

          /** The reserved session id. */
          private Integer sessionId;

          /** The worker thread id running the invoke. */
          private Integer threadId;

          /**
           * Private constructor of the standard invoke wrapper.
           *
           * @param    taskType
           *           The type of the invoke.
           * @param    connectionId
           *           The appserver connection id.
           * @param    requestId
           *           The ID of the request.
           * @param    clientDrivenSessionExecutionTimeout
           *           The maximum allowed time for this invocation to complete. In milliseconds.
           *           Use <code>0</code> to disable the timeout.
           * @param    isPersistentProcRun
           *           <code>true</code> if the invoke should run in persistent mode.
           */
          private StandardInvoke(MultiSessionTaskType taskType,
                                 String connectionId,
                                 int requestId,
                                 long clientDrivenSessionExecutionTimeout,
                                 boolean isPersistentProcRun)
          {
             this.taskType = taskType;
             this.mainConnectionId = connectionId;
             this.requestId = requestId;
             this.clientDrivenSessionExecutionTimeout = clientDrivenSessionExecutionTimeout;
             this.isPersistentProcRun = isPersistentProcRun;
             this.result = new AppServerInvocationResult();
             this.agentId = MultiSessionAgent.SINGLE_AGENT_ID;
          }

          /**
           * Validates if the main connection is running on a live appserver.
           *
           * @param    isInCall
           *           Flag to indicate if the invoke is RUN IN handle call, or method invoke.
           *
           * @return   <code>true</code> if the main connection is found on a running appserver.
           */
          private boolean validateMainConnection(boolean isInCall)
          {
             mainConnection = connections.get(mainConnectionId);
             if (mainConnection == null || mainConnection.isDisconnecting())
             {
                boolean isReestablished = false;
                synchronized (timedOutConnections)
                {
                   if (timedOutConnections.containsKey(mainConnectionId))
                   {
                      MultiSessionClientConnection timedOutConn = timedOutConnections.remove(mainConnectionId);
                      if (timedOutConn.isSessionFree() && !isInCall)
                      {
                         mainConnection = createNewClientConnection(timedOutConn.getAppserverName(),
                                                                    null,
                                                                    timedOutConn.isSessionFree(),
                                                                    timedOutConn.isForWebService());
                         mainConnection.disableLegacyErrors(timedOutConn.hasLegacyErrorsDisabled());
                         isReestablished = true;
                      }
                      else
                      {
                         result.setError(
                            new ErrorConditionException("Failure reading response from Application Server.",
                                                        14810),
                            timedOutConn.hasLegacyErrorsDisabled());
                         return false;
                      }
                   }
                }
                if (!isReestablished)
                {
                   result.setError(new ErrorConditionException("Server is not connected.", 5537));
                   return false;
                }
             }

             String appserverName = mainConnection.getAppserverName();
             if (!startedAppservers.containsKey(appserverName))
             {
                result.setError(new NumberedException("Application server connect failure.", 5468));
                return false;
             }

             msaDef = MultiSessionAppserverDefinition.get(appserverName);
             msaAppserver = startedAppservers.get(appserverName);
             if (!msaAppserver.isRunning())
             {
                result.setError(new NumberedException("Application server connect failure.", 5468));
                return false;
             }
             return true;
          }

          /**
           * Validates the main connection exists on a live appserver, if this is not already done, and determines
           * the invoke connection.
           *
           * @param    cfg
           *           The invoke configuration.
           * @param    inProcResId
           *           The resource id of the in-handle.
           *
           * @return   <code>true</code> if the invoke connection is found on a running appserver.
           */
          private boolean determineConnection(InvokeConfig cfg, String inProcResId)
          {
             isConnectionValidationDone.set(true);

             boolean isInCall = cfg.getInHandle() != null || inProcResId != null;

             if (!validateMainConnection(isInCall))
             {
                return false;
             }

             String appserverName = msaDef.getAppserverName();
             isPersistentProcRun = !cfg.isClass() && cfg.getProcedureHandle() != null;

             if (isPersistentProcRun && mainConnection.isSessionFree())
             {
                // in Session-free persistent procs create their own client connections to bind
                invokeConnection = createNewClientConnection(appserverName, mainConnection, true);
                mainConnection.addChildConnection(invokeConnection);
             }
             else if (mainConnection.isSessionFree() && isInCall && !cfg.isSingleRun() && !cfg.isSingleton())
             {
                // in Session-free in-handle for persistent proc should switch the connection
                String persistentProcId = inProcResId != null ?
                   inProcResId :
                   ((ProxyProcedureWrapper) cfg.getInHandle().getResource()).getResourceId();
                invokeConnection = mainConnection.findConnectionFor(persistentProcId);
                if (invokeConnection == null)
                {
                   result.setError(new ErrorConditionException("Server is not connected.", 5537),
                                   mainConnection.hasLegacyErrorsDisabled());
                   return false;
                }
             }
             else if (mainConnection.isSessionFree() && cfg.isAsynchronous())
             {
                // TODO: if it's the first async request running on the connection, use the main connection
                // in Session-free async procs create their own client connections to bind
                invokeConnection = createNewClientConnection(appserverName, mainConnection, true);
                mainConnection.addChildConnection(invokeConnection);
             }
             else
             {
                invokeConnection = mainConnection;
             }
             return true;
          }

          /**
           * Validates the main connection exists on a live appserver, if this is not already done, and determines
           * the invoke connection.
           *
           * @param    resId
           *           The external procedure resource id.
           *
           * @return   <code>true</code> if the invoke connection is found on a running appserver.
           */
          private boolean determineConnection(String resId)
          {
             isConnectionValidationDone.set(true);

             boolean isInCall = resId != null;
             if (!validateMainConnection(isInCall))
             {
                return false;
             }

             if (!isInCall)
             {
                invokeConnection = mainConnection;
                return true;
             }

             isPersistentProcRun = true;

             // in Session-free persistent procs should switch to their own connection
             invokeConnection = mainConnection.findConnectionFor(resId);
             if (invokeConnection == null)
             {
                throw new IllegalArgumentException("The procedure can not be accessed on this connection!");
                //result.setError(new NumberedException("Invalid or inappropriate handle value given to " +
                // "RUN...IN statement. Procedure '" + fileName + "':" + line + ".", 2128));
                //return false;
             }

             return true;
          }

          /**
           * Validates the main connection exists on a live appserver, if this is not already done, and determines
           * the invoke connection.
           *
           * @return   <code>true</code> if the invoke connection is found on a running appserver.
           */
          private boolean determinePseudoPersistentConnection()
          {
             isConnectionValidationDone.set(true);

             if (!validateMainConnection(false))
             {
                return false;
             }

             isPersistentProcRun = true;

             if (mainConnection.isSessionFree())
             {
                // in Session-free persistent procs create their own client connections to bind
                invokeConnection = createNewClientConnection(msaDef.getAppserverName(), mainConnection, true);
                mainConnection.addChildConnection(invokeConnection);
             }
             else
             {
                invokeConnection = mainConnection;
             }
             return true;
          }

          /**
           * Creates and executes a MSA task wrapper that reserves an MSA session and sets up the invoke
           * details in the result.
           *
           * @param    runnable
           *           The runnable to be executed on the worker.
           *
           * @return   A {@link AppServerInvocationResult} instance, which holds the code for the remote
           *           object, all updated OUTPUT/INPUT-OUTPUT arguments and any returned value.
           */
          private AppServerInvocationResult exec(Runnable runnable)
          {
             if (!isConnectionValidationDone.get() && !determineConnection(null))
             {
                return result;
             }

             invokeConnection.setLastUseTime();

             CancellableCompositeTask<AppServerInvocationResult> invokeTask =
                new CancellableCompositeTask<>(new MsaCancellableCallable()
                {
                   @Override
                   public AppServerInvocationResult call()
                   throws Exception
                   {
                      sessionId = reserveSession(msaAppserver, invokeConnection);

                      if (isCancelled() || sessionId == MultiSessionAppserver.INVALID_SESSION_ID)
                      {
                         String errMsg = getCancelCause() == CancelCause.SEND_STOP ? "" : ERR_NO_CONNECTIONS;
                         result.setError(new ErrorConditionException(errMsg));
                         return result;
                      }

                      threadId = (int) Thread.currentThread().getId();

                      result.setAgentId(MultiSessionAgent.SINGLE_AGENT_ID);
                      result.setConnectionID(mainConnectionId);

                      if (!invokeConnection.getId().equals(mainConnectionId))
                      {
                         result.setChildConnectionID(invokeConnection.getId());
                      }

                      result.setConnectionModel(invokeConnection.getModel());
                      result.setSessionId(sessionId);
                      result.setThreadId(threadId);
                      result.setRequestId(String.valueOf(requestId));

                      msaAppserver.runInSessionContext(invokeConnection,
                                                       sessionId,
                                                       result,
                                                       runnable,
                                                       getIsRunningSupplier(),
                                                       getCancelledSupplier(),
                                                       getTaskReceiver());

                      return result;
                   }
                }, taskType, requestId);

             if (clientDrivenSessionExecutionTimeout > DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT &&
                LOG.isLoggable(Level.FINE))
             {
                LOG.fine("MSA request running with a custom timeout " + clientDrivenSessionExecutionTimeout +
                            " milliseconds.");
             }

             return handleExec(invokeTask,
                               result,
                               invokeConnection,
                               msaDef.getRequestWaitTimeout(),
                               clientDrivenSessionExecutionTimeout > DISABLE_CLIENT_DRIVEN_SESS_EXEC_TIMEOUT ?
                                  (int) clientDrivenSessionExecutionTimeout :
                                  msaDef.getSessionExecutionTimeLimit());
          }
       }

       /** Cancellable callable for the multi-session agent appservers. */
       private static abstract class MsaCancellableCallable
       extends CancellableCallable<AppServerInvocationResult>
       {
          /**
           * Manages the Future for reserving the session.
           *
           * @param    msaAppserver
           *           The MSA appserver.
           * @param    invokeConnection
           *           The invoke connection.
           *
           * @return   The id of the reserved session.
           *
           * @throws   ExecutionException
           *           If the computation threw an exception
           * @throws   InterruptedException
           *           If the current thread was interrupted while waiting.
           */
          Integer reserveSession(MultiSessionAppserver msaAppserver,
                                 MultiSessionClientConnection invokeConnection)
          throws ExecutionException,
                 InterruptedException
          {
             Future<Integer> reserveSessionFuture = msaAppserver.reserveSession(invokeConnection);
             setCurrentlyRunningFuture(reserveSessionFuture);
             int sessionId = reserveSessionFuture.get();
             setCurrentlyRunningFuture(null);
             return sessionId;
          }
       }

       /**
        * Stores global data relating to the state of the current MSA context.
        */
       private static class WorkArea
       {
          /** Flag to show a startup procedure is running. */
          private boolean isStartupProc;

          /** The id of the agent the current context is running in. */
          private Integer agentId;

          /** The id of the MSA session the current context is running in. */
          private Integer sessionId;

          /** The current client connection. */
          private MultiSessionClientConnection connection;

          /** Stacktrace of the original thread of the task. Used for debugging. */
          private List<StackTraceElement[]> debugStackTrace;

          /**
           * Constructor.
           */
          WorkArea()
          {
          }
       }
    }