AgentPool.java

/*
** Module   : AgentPool.java
** Abstract : Definition and implementation of agent pools (based on appserver's operating mode).
**
** Copyright (c) 2013-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description----------------------------------------
** 001 CA  20130529 Created initial version.
** 002 CA  20130628 Fixed context-local data reset for State-reset operating mode.
** 003 CA  20130704 Reworked appserver launching - implemented by the AppServerLauncher class.
**                  Appserver definition was moved to its own class.
** 004 CA  20130823 Changes related to support async requests.
** 005 CA  20140216 Fixed a deadlock related to reseting the agent's context, when there is a 
**                  disconnect procedure.
** 006 CA  20140224 Pass a handle when running the connect procedure.
** 007 CA  20140326 The remote persistent procedures need to be deleted by the Agent which created
**                  them.
** 008 CA  20140826 Fixed agent context cleanup if the P2J session is terminated before the 
**                  appserver connection is established.
** 009 EVL 20160225 Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
** 010 OM  20160425 Project awareness (support for running multiple projects in same JVM).
** 011 CA  20180503 Added SESSION:SERVER-CONNECTION-ID support.
** 012 CA  20190612 Added APIs to create remote legacy objects and invoke methods on them.
** 013 CA  20190628 Fixed STATE-FREE issues, when the Agent is QUIT or otherwise terminated.
** 014 CA  20190811 Fixed connection ID for virtual sessions.
** 015 CA  20191211 For multi-session appserver, the connection ID is using 'abcdefghijklm_abcdefgh' format.
**     CA  20200110 Fix for agent trimming, when executed from other FWD clients.
** 016 CA  20200427 Added OERequestInfo support.
**     CA  20200514 Fixed error handling when no agent is available.
** 017 CA  20201003 Replaced Guava identity HashSet with Collections.newSetFromMap(IdentityHashMap).
** 018 CA  20211112 Fixed a bug in removeProcedure - when the connection ID does not exist.
**     CA  20211214 For State-free appservers, the agents can be bound to remote persistent procedures; this
**                  binding must be made using the agent's ID, as a client can invoke multiple remote 
**                  persistent procedures, and using the procedure's ID for this binding can lead to 
**                  collisions, as the pair (connection ID, procedure ID) is not guaranteed to be unique for
**                  a connection.
**     RFB 20211216 Minor javadoc errors cleaned up.
**     CA  20220104 Fixed Stateless appserver mode, which got broken in CA/20211214 - now the agent tracks any
**                  persistent procedure ran over it, 
**     CA  20220111 Fixed SESSION:SERVER-CONNECTION-CONTEXT - this value must be shared to any agent running
**                  requests for a connection.
**     CA  20220208 For calls originating from outside the FWD server, do not serialize the legacy OO errors.
**     CA  20220514 The active session is now set for leaf sessions, too - this is required for a change in  
**                  RemoteObject.obtainInstance, where first a check is done for an existing local proxy, and 
**                  after that a network instance is obtained (a requirement for the server-side OS resources
**                  support, like memptr).
**     CA  20220630 Fixed a NPE for SERVER-CONNECTION-CONTEXT setter.
**     CA  20221031 Added JMX instrumentation to monitor appserver agents state.
**     CA  20220630 Fixed a NPE for SERVER-CONNECTION-CONTEXT setter.
**     CA  20230130 Added options to trim or restart the agents in an appserver.
** 019 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 020 HC  20240222 Enabled JMX on FWD Client.
** 021 GBB 20240318 Session model as enum.
** 022 CA  20240729 Fixed deactivate on procedure delete: the deactivate procedure must be invoked before the
**                  agent gets released.
** 023 CA  20240821 Deleting the agent's 'bound persistent procedure' must deactivate and unbind the agent in
**                  Stateless mode, too.
** 024 GBB 20241010 Class made public. Adding getter for all agents.
** 025 GBB 20250403 Reflect the changes in the appserver definition.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util;

import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.logging.*;

import com.goldencode.p2j.*;
import com.goldencode.p2j.jmx.*;
import com.goldencode.p2j.main.*;
import com.goldencode.p2j.net.SessionManager;
import com.goldencode.p2j.security.SecurityManager;
import com.goldencode.p2j.util.Agent.InvocationModes;
import com.goldencode.p2j.util.appserver.*;
import com.goldencode.p2j.util.logging.*;

/**
 * The main class for agent pooling.  Based on the appserver's operating mode, the 
 * {@link #newConnection()} call will return a:
 * <ul>
 * <li> {@link BoundPool} for State-reset and State-aware modes.</li>
 * <li> {@link UnboundPool} for State-free and Stateless modes.</li>
 * </ul>
 * Agents are extracted from {@link #mainPool} as connection or procedure requests are incoming
 * and put back in the pool only after the connection terminates or it becomes unbound.
 */
public abstract class AgentPool
{
   /** Logger. */
   private static final CentralLogger LOG = CentralLogger.get(AgentPool.class);

   /** Map of connection IDs to appserver name. */
   private static final Map<String, String> idsToAppserver = new HashMap<String, String>();

   /** The pool of agents.*/
   private final Set<Agent> mainPool = Collections.newSetFromMap(new IdentityHashMap<>());
   
   /** The map of agents. */
   private final Map<Integer, Agent> agents = new ConcurrentHashMap<>();
   
   /** The value of SESSSION:SERVER-CONNECTION-CONTEXT set for each connection. */
   private final Map<String, String> serverConnectionContext = new ConcurrentHashMap<>();
   
   /** 
    * The remote persistent procedures exposed as proxy procedures to the requester.  This map is also used
    * on the agent side to determine if a procedure was ran remote or not.
    */
   protected final Map<Object, Agent> persistentProcedures = new IdentityHashMap<>();

   /** All started agents. */
   private final Set<Agent> all = Collections.newSetFromMap(new IdentityHashMap<>());
   
   /** Set of connection IDs which can't throw legacy OO errors on the remote side. */
   private final Set<String> disabledLegacyErrors = new HashSet<>();
   
   /** The appserver definition. */
   private final ClassicAppserverDefinition appDef;
   
   /** 
    * The export list (defined with SESSION:EXPORT([])), which specifies the remote 
    * procedures a client can execute in the current appserver session.
    */
   private String exports = null;
   
   /** The generator for Agent IDs. */
   private int agentIdGenerator = 0;
   
   /**
    * Build a new agent pool, for the specified appserver.
    * 
    * @param    def
    *           The appserver definition.
    */
   private AgentPool(ClassicAppserverDefinition def)
   {
      this.appDef = def;
      FwdServerJMX.register(new AgentMonitor(this), "AppServer_" + def.getAppserverName());
   }

   /**
    * Getter for {@link #all}.
    *
    * @return   All started agents.
    */
   public Set<Agent> getAll()
   {
      return new HashSet<>(all);
   }
   
   /**
    * Get the appserver name used to established the given connection.
    * 
    * @param    connectionId
    *           The connection ID.
    *           
    * @return   See above.
    */
   static String getAppserver(String connectionId)
   {
      synchronized (idsToAppserver)
      {
         return idsToAppserver.get(connectionId);
      }
   }
   
   /**
    * Instantiate a new agent pool for the given appserver, based on the operating mode.
    * 
    * @param    appServer
    *           The appserver's name.
    *           
    * @return   A new pool instance.
    * 
    * @throws   IllegalArgumentException
    *           If the operating mode can not be resolved to one of {@link AppServerOperatingMode}
    *           constants.
    */
   static AgentPool newPool(String appServer)
   {
      ClassicAppserverDefinition appDef = ClassicAppserverDefinition.get(appServer);
      switch (appDef.getOperatingMode())
      {
         case STATE_AWARE:
            return new BoundPool(appDef, false);
         case STATE_RESET:
            return new BoundPool(appDef, true);
         case STATELESS:
            return new UnboundPool(appDef, false);
         case STATE_FREE:
            return new UnboundPool(appDef, true);
      }
   
      throw new IllegalArgumentException(
         "Illegal operating mode: " + appDef.getOperatingMode() + ".");
   }

   /**
    * Acquire a new agent from the pool to handle requests incoming via the given connection ID.
    * 
    * @param    id
    *           The connection ID.
    * @param    code
    *           The code of a persistent external procedure to which the agent is associated. May
    *           be <code>null</code>.
    * @param    agentId
    *           The agent ID which created the given procedure code, a negative number means the procedure
    *           code will not be used.
    * 
    * @return   See above.
    * 
    * @throws   NumberedException
    *           If there was no agent available.
    */
   abstract Agent acquireAgent(String id, String code, int agentId)
   throws NumberedException;

   /**
    * Release the given agent and place it back in the pool. This will handle any bound-related
    * constraints.
    * 
    * @param    id
    *           The connection ID.
    * @param    agent
    *           The agent which needs to be released.
    */
   abstract void releaseAgent(String id, Agent agent);

   /**
    * Establish a new connection.
    * 
    * @return   The connection ID.
    * 
    * @throws   NumberedException
    *           If the connection could not be established.
    */
   abstract String newConnection()
   throws NumberedException;

   /**
    * Determine if the agent must be unbound and deactivate procedure invoked, when there are no more
    * persistent procedures bound to it.
    * 
    * @param    id
    *           The connection ID.
    * @param    referent
    *           The persistent procedure.
    *           
    * @return   <code>true</code> if the deactivate can be invoked and the agent unbound.
    */
   abstract boolean deactivateAgentOnRemoveProcedure(String id, Object referent);

   /**
    * Disable legacy OO errors thrown on this connection.
    * 
    * @param    id
    *           The appserver connection id. 
    */
   void disableLegacyErrors(String id)
   {
      disabledLegacyErrors.add(id);
   }
   
   /**
    * Check if the given connection ID should suppress legacy OO errors.
    * 
    * @param    id
    *           The appserver connection id.
    *           
    * @return   See above.
    */
   boolean isDisabledLegacyErrors(String id)
   {
      return disabledLegacyErrors.contains(id);
   }
   
   /**
    * Acquire a new agent from the pool to handle requests incoming via the given connection ID.
    * 
    * @param    id
    *           The connection ID.
    *           
    * @return   See above.
    * 
    * @throws   NumberedException
    *           If there was no agent available.
    */
   Agent acquireAgent(String id)
   throws NumberedException
   {
      return acquireAgent(id, null, -1);
   }

   /**
    * Get the agent for the given ID.
    * 
    * @param    agentId
    *           The agent ID.
    */
   Agent getAgent(int agentId)
   {
      return agents.get(agentId);
   }
   
   /**
    * Get the next connection ID.
    * 
    * @return   See above.
    */
   String nextId()
   {
      if (appDef.isMultiSession())
      {
         return AppServerManager.nextContextId(true);
      }
      
      SecurityManager sm = SecurityManager.getInstance();
      SessionManager sessMgr = SessionManager.get();
      boolean virtual = sessMgr.isVirtualSession();
      boolean local = !sessMgr.isLeaf();
      
      // In 4GL, the connection ID uses broker information, as the appserver client can connect
      // via a broker which routes the connection to the appserver.
      // In FWD, the broker is a Java process running remote, to which FWD Client launching is
      // delegated (if the FWD user is configured for broker launching).
      // So, as 4GL docs state that the connection ID is just use to uniquely identify the client
      // (and the 4GL dev can't rely on this ID to identify host/port), we are using the connected
      // client's remote host/port.
      
      String host      = (virtual ? "virtual(" + sm.getUserId() + ")" 
                                  : (local ? "local" : sm.getPeerHost()));
      int    port      = (virtual || local ? 0 : sm.getPeerPort());
      String appServer = appDef.getAppserverName();
      
      do
      {
         String uuid = UUID.randomUUID().toString();
         String next = String.format("%s::%s::%s::%s", host, appServer, port, uuid);
   
         synchronized (idsToAppserver)
         {
            if (!idsToAppserver.containsKey(next))
            {
               return next;
            }
         }
      }
      while (true);
   }

   /**
    * Get the exports for this appserver.  If set, only the external procedures which match these
    * exports will be available for invocation.
    * 
    * @return   See above.
    */
   synchronized character getExports()
   {
      return new character(exports);
   }
   
   /**
    * Set the exports for this appserver.
    * 
    * @param    exports
    *           The appserver's exports. 
    */
   void setExports(final character exports)
   {
      this.exports = (exports.isUnknown() ? null : exports.toStringMessage());
   }
   
   /**
    * Get the name of the appserver handled by this pool.
    * 
    * @return   See above.
    */
   String getName()
   {
      return appDef.getAppserverName();
   }

   /**
    * Get the appserver's propath.
    * 
    * @return   See above.
    */
   String getPropath()
   {
      return appDef.getPropath();
   }
   
   /**
    * Get the startup procedure for the appserver handled by this pool.
    * 
    * @return   See above.
    */
   String getStartup()
   {
      return appDef.getStartup();
   }

   /**
    * Get the startup parameters for the appserver handled by this pool.
    * 
    * @return   See above.
    */
   String getStartupParameter()
   {
      return appDef.getStartupParameter();
   }

   /**
    * Get the shutdown procedure for the appserver handled by this pool.
    * 
    * @return   See above.
    */
   String getShutdown()
   {
      return appDef.getShutdown();
   }

   /**
    * Get the activate procedure for the appserver handled by this pool.
    * 
    * @return   See above.
    */
   String getActivate()
   {
      return appDef.getActivate();
   }

   /**
    * Get the deactivate procedure for the appserver handled by this pool.
    * 
    * @return   See above.
    */
   String getDeactivate()
   {
      return appDef.getDeactivate();
   }

   /**
    * Establish a new appserver connection. This will invoke any connect procedure for this 
    * appserver, to which the given <code>user</code>, <code>pwd</code> and <code>serverInfo</code>
    * parameters will be passed.
    * <p>
    * The appserver connection must be configured to start under the given P2J-level subject ID.
    *  
    * @param    sessionTerminated
    *           A special var to inform us if the session has been terminated.  The session state
    *           can't be invoked directly from here, as a deadlock might occur.
    * @param    subjectId
    *           The subject ID associated with this appserver.
    * @param    sessionFree
    *           Flag indicating if the connection request is for a session-free operating mode.
    * @param    user
    *           The first parameter to pass to the connect procedure, if any.
    * @param    pwd
    *           The second parameter to pass to the connect procedure, if any.
    * @param    serverInfo
    *           The third parameter to pass to the connect procedure, if any.
    * @param    requestInfo
    *           The remote {@link com.goldencode.p2j.oo.lang.OerequestInfo} details, as set by
    *           {@link com.goldencode.p2j.oo.lang.OerequestInfo#toArray()}.
    *           
    * @return   An {@link AppServerInvocationResult} instance with any result or caught errors.
    */
   AppServerInvocationResult connect(AtomicBoolean sessionTerminated,
                                     String        subjectId,
                                     boolean       sessionFree,
                                     String        user,
                                     String        pwd,
                                     String        serverInfo,
                                     Object[]      requestInfo)
   {
      AppServerInvocationResult result = new AppServerInvocationResult();
   
      AppServerOperatingMode operatingMode = getOperatingMode();
      if (operatingMode.isSessionFree() != sessionFree)
      {
         final String msg =
            "The %s connection failed because the AppServer is in a %s operating mode (%s)";
         final String err = String.format(msg,
                                          sessionFree ?
                                             AppServerSessionModel.SESSION_FREE.toString() :
                                             AppServerSessionModel.SESSION_MANAGED.toString(),
                                          operatingMode.isSessionFree() ?
                                             AppServerSessionModel.SESSION_FREE.toString() :
                                             AppServerSessionModel.SESSION_MANAGED.toString(),
                                          operatingMode);
         result.setError(new NumberedException(err, 13511));
         return result;
      }
   
      String id = null;
      try
      {
         id = newConnection();
      }
      catch (NumberedException e)
      {
         result.setError(e);
         return result;
      }
      
      synchronized(idsToAppserver)
      {
         idsToAppserver.put(id, appDef.getAppserverName());
      }
      
      String connect = appDef.getConnect();

      // invoke the connect procedure only if the session is not yet terminated.
      if (!sessionTerminated.get()       && 
          !operatingMode.isSessionFree() && 
          connect != null && connect.length() > 0)
      {
         handle hConn = new handle();
         Agent agent;
         try
         {
            agent = acquireAgent(id);
         }
         catch (NumberedException e)
         {
            result.setError(e);
            return result;
         }
         
         agent.invoke(requestInfo, result, 
                      id, Agent.InvocationModes.AGENT_WAIT, connect, hConn, "Connect", 
                      "III", new character(user), new character(pwd), new character(serverInfo));
         releaseAgent(id, agent);
         
         if (!sessionTerminated.get() && result.getError() != null)
         {
            // in case of error, disconnect, but only if the session has not been terminated
            disconnect(id);
   
            id = null;
         }
      }
      
      if (id != null && sessionTerminated.get())
      {
         // there is a special behavior here: if the session is terminated while or before
         // establishing the appserver connection, the agent is released without calling the 
         // disconnect procedure.
         terminateConnection(id, false);

         id = null;
      }
      
      result.setConnectionID(id);
      
      // regardless of OK/error, the result is sent
      return result;
   }

   /**
    * Disconnect the appserver connection, with the given ID.  It will invoke any configured 
    * disconnect procedure, but will not return any errors or values to the caller.
    * 
    * @param    id
    *           The connection ID to disconnect.
    */
   void disconnect(String id)
   {
      disabledLegacyErrors.remove(id);
      
      AppServerOperatingMode operatingMode = getOperatingMode();
      
      String disconnect = appDef.getDisconnect();
      if (!operatingMode.isSessionFree() && disconnect != null && disconnect.length() > 0)
      {
         boolean stateless = (appDef.getOperatingMode() == AppServerOperatingMode.STATELESS);
         AppServerInvocationResult result = new AppServerInvocationResult();
         
         Agent agent;
         try
         {
            agent = acquireAgent(id);
         }
         catch (NumberedException e)
         {
            result.setError(e);
            return;
         }
         // call the disconnect procedure.
         InvocationModes mode = (stateless ? InvocationModes.AGENT_WAIT 
                                           : InvocationModes.AGENT_NO_WAIT);
         agent.invoke(result, id, mode, disconnect, null, "Disconnect", null);
      }
      else
      {
         terminateConnection(id, false);
      }
   }

   /**
    * Start listening for appserver-related commands.
    */
   void start()
   {
      // this is a new Agent for the appServer. do not allow to start more than maxAgents
      int max = appDef.getMaxAgents();
      if (size() >= max)
      {
         if (LOG.isLoggable(Level.WARNING))
         {
            LOG.warning("Pool capacity of " + max + " reached, no more agents will be started.");
         }

         // just terminate the client, nothing else to do
         return;
      }
      
      // from this point forward we use the directory settings associated with this project
      if (appDef.getProjectToken() != null)
      {
         Utils.setProjectToken(appDef.getProjectToken());
      }
      
      // add it to the pool and let it listen for commands
      Agent agent = new Agent(this, agentIdGenerator++);
      agents.put(agent.getAgentId(), agent);
      
      poolAgent(agent, true);
      try
      {
         agent.listen();
      }
      finally
      {
         // the agent was terminated, remove it from the pool
         removeAgent(agent);
      }
   }

   /**
    * Send the terminate command to all agents in this pool.
    */
   void terminateAll()
   {
      Set<Agent> copy = null;
      synchronized (all)
      {
         copy = new HashSet<Agent>(all);
      }

      for (Agent agent : copy)
      {
         agent.terminate();
      }
   }

   /**
    * Terminate the connection with the given ID.
    * 
    * @param    id
    *           The connection ID.
    * @param    running
    *           Flag indicating the Agent is already processing a command.
    */
   void terminateConnection(String id, boolean running)
   {
      synchronized(idsToAppserver)
      {
         idsToAppserver.remove(id);
      }
      
      // cleanup any SESSION:SERVER-CONNECTION-CONTEXT value.
      serverConnectionContext.remove(id);
   }

   /**
    * Get the operating mode for this appserver.
    * 
    * @return   See above.
    */
   AppServerOperatingMode getOperatingMode()
   {
      return appDef.getOperatingMode();
   }
   
   /**
    * Check if this definition is for multi-session agent appserver.
    * 
    * @return    See above.
    */
   boolean isMultiSession()
   {
      return appDef.isMultiSession();
   }

   /**
    * Add an agent back to the {@link #mainPool}.
    * 
    * @param    agent
    *           The agent to add back to the pool.
    * @param    newlyCreated
    *           When this flag is <code>true</code>, the agent is added to the {@link #all global}
    *           pool too.
    */
   void poolAgent(Agent agent, boolean newlyCreated)
   {
      agent.unbind();
      
      synchronized (mainPool)
      {
         mainPool.add(agent);
         mainPool.notify();
      }
   
      if (newlyCreated)
      {
         synchronized (all)
         {
            all.add(agent);
            all.notify();
         }
      }
      else
      {
         agent.setConnectionId(null);
      }
   }

   /**
    * Check if this pool is empty.
    * 
    * @return   <code>true</code> if the pool is empty, <code>false</code> otherwise.
    */
   boolean isEmpty()
   {
      synchronized (all)
      {
         return all.isEmpty();
      }
   }
   
   /**
    * Remove the given agent entirely from the pool.
    * 
    * @param    agent
    *           The agent which was terminated and needs to be removed.
    */
   void removeAgent(Agent agent)
   {
      synchronized (all)
      {
         all.remove(agent);
         all.notify();
      }
      
      synchronized (mainPool)
      {
         mainPool.remove(agent);
      }
      
      agents.remove(agent.getAgentId());
   }

   /**
    * Get the pool size.
    * 
    * @return   See above.
    */
   int size()
   {
      synchronized (all)
      {
         return all.size();
      }
   }
   
   /**
    * Get the number of available agents, waiting for work, in {@link #mainPool}.
    * 
    * @return   See above.
    */
   int available()
   {
      synchronized (mainPool)
      {
         return mainPool.size();
      }
   }
   
   /**
    * Check if the given persistent procedure was started as a remote procedure with the appserver associated 
    * with this pool.
    * 
    * @param    referent
    *           The persistent procedure.
    *           
    * @return   See above.
    */
   boolean hasProcedure(Object referent)
   {
      synchronized (persistentProcedures)
      {
         return persistentProcedures.keySet().contains(referent);
      }
   }
   
   /**
    * Register the given persistent procedure with the {@link Agent} which created it.
    * 
    * @param    referent
    *           The persistent procedure.
    * @param    agent
    *           The agent which created the persistent procedure.
    */
   void addProcedure(Object referent, Agent agent)
   {
      synchronized (persistentProcedures)
      {
         persistentProcedures.put(referent, agent);
      }
   }
   
   /**
    * De-register the persistent procedure from {@link #persistentProcedures}.
    * 
    * @param    id
    *           The connection ID.
    * @param    referent
    *           The persistent procedure.
    */
   final void removeProcedure(String id, Object referent)
   {
      boolean unbindAndDeactivate = deactivateAgentOnRemoveProcedure(id, referent);

      Agent agent = null;
      synchronized (persistentProcedures)
      {
         agent = persistentProcedures.remove(referent);
      }
      
      if (unbindAndDeactivate && agent != null)
      {
         // if there was a bound agent, unbind it and call deactivate
         agent.deactivate(id);

         // and also released
         releaseAgent(id, agent);
      }
   }
   
   /**
    * Get the definition of this appserver.
    * 
    * @return   The {@link #appDef appserver definition}.
    */
   ClassicAppserverDefinition getDef()
   {
      return appDef;
   }
   
   /**
    * Terminate {@link #all} running agents and start the same number of agents, to refresh their context.  
    */
   void restartAgents()
   {
      int numAgents = 0;
      synchronized (all)
      {
         numAgents = all.size();
      }
      
      terminateAll();
      
      String appServer = appDef.getAppserverName();
      for (int i = 0; i < numAgents; i++)
      {
         AppServerLauncher.launch(appServer);
      }
   }
   
   /**
    * Trim the running agents so the pool size is as close to 
    * {@link ClassicAppserverDefinition#getMinAgents()} as possible.
    */
   void trimAgents()
   {
      final int size = appDef.getMinAgents();
      // use a copy, as 
      Set<Agent> copy = null;
      synchronized (all)
      {
         copy = new HashSet<Agent>(all);
      }

      int n = copy.size() - size;
      // check if we can terminate any agents
      if (n <= 0)
      {
         return;
      }

      for (Agent a : copy)
      {
         synchronized (mainPool)
         {
            // if this agent is not busy, terminate it
            if (mainPool.contains(a))
            {
               // remove it from main pool first, so it can't be picked by other connections
               mainPool.remove(a);
               
               // terminate
               a.terminate();
               
               n = n - 1;
            }
            
            // we either reach the minimum pool size or terminate the computed number of agents. 
            if (mainPool.size() - size <= 0)
            {
               break;
            }
         }

         // exit if no more trimming possible
         if (n == 0)
         {
            break;
         }
      }
   }

   /**
    * Extract an agent from the main pool; if non available, attempt to start one. If the max
    * number of running agents is reached and all agents are busy, return <code>null</code>.
    * 
    * @return   The extracted agent or <code>null</code> if all agents are busy.
    */
   Agent getFromMainPool()
   {
      Agent agent = null;
      long timeout = appDef.getRequestTimeout() * 1000;
      synchronized (mainPool)
      {
         if (mainPool.size() == 0 && timeout > 0)
         {
            try
            {
               mainPool.wait(timeout);
            }
            catch (InterruptedException e)
            {
               // ignore
            }
         }
   
         if (mainPool.size() > 0)
         {
            agent = mainPool.iterator().next();
            mainPool.remove(agent);
         }
      }

      int prevSize = size();
      if (agent == null && prevSize < appDef.getMaxAgents())
      {
         // automatically start an agent, until maxAgents is reached
         AppServerLauncher.launch(appDef.getAppserverName());
         
         // if an agent was started, get it from the main pool.
         if (size() > prevSize)
         {
            agent = getFromMainPool();
         }
      }
      
      return agent;
   }
   
   /**
    * Get the SERVER-CONNECTION-CONTEXT attribute of the specified connection.
    * 
    * @param    connectionID
    *           The connection ID for this appserver.
    *           
    * @return   See above.
    */
   String getServerConnectionContext(String connectionID)
   {
      String ctxt = serverConnectionContext.get(connectionID);
      return ctxt == null ? "" : ctxt;
   }
   
   /**
    * Set the SERVER-CONNECTION-CONTEXT attribute of the specified connection.
    * 
    * @param    connectionID
    *           The connection ID for this appserver.
    * @param    serverConnectionContext
    *           The attribute's value.
    */
   void setServerConnectionContext(String connectionID, String serverConnectionContext)
   {
      this.serverConnectionContext.put(connectionID, 
                                       serverConnectionContext == null ? "" : serverConnectionContext);
   }

   /**
    * Class implementing the behavior for STATE-RESET and STATE-AWARE operating modes.  In these
    * modes, an agent is bound to the connection when the connection is established.
    */
   static class BoundPool
   extends AgentPool
   {
      /** The reserved agent, for each connection. */
      private final Map<String, Agent> connections = new HashMap<String, Agent>();

      /** Flag indicating this is STATE-RESET mode. */
      private final boolean reset;
      
      /**
       * Build a new agent pool, for the specified appserver.
       * 
       * @param    appDef
       *           The appserver definition.
       * @param    reset
       *           Flag indicating this is a STATE-RESET mode.
       */
      BoundPool(ClassicAppserverDefinition appDef, boolean reset)
      {
         super(appDef);
         this.reset = reset;
      }
      
      /**
       * Get the agent associated with the given connection id.
       * 
       * @param    id
       *           The connection ID.
       * @param    code
       *           The code of a persistent external procedure to which the agent is associated. 
       *           Not used for bound pools (as async requests are not possible).
       * @param    agentId
       *           When non-negative, the agent ID which created the persistent procedure.
       * @return   See above.
       * 
       * @throws   NumberedException
       *           If there was no agent available.
       */
      Agent acquireAgent(String id, String code, int agentId)
      throws NumberedException
      {
         synchronized (connections)
         {
            return connections.get(id);
         }
      }
      
      /**
       * This is a no-op, as the agent is released only when the connection is terminated.
       * 
       * @param    id
       *           The connection ID.
       * @param    agent
       *           The agent which needs to be released.
       */
      void releaseAgent(String id, Agent agent)
      {
         // no-op, agent is kept bound for the duration of the connection
      }
      
      /**
       * Establish a new connection.
       * 
       * @return   The connection ID.
       * 
       * @throws   NumberedException
       *           If the connection could not be established.
       */
      String newConnection()
      throws NumberedException
      {
         String id = nextId();
         // must reserve one agent for this bound connection
         Agent agent = getFromMainPool();
         if (agent == null)
         {
            throw new NumberedException("No servers available");
         }
         
         synchronized (connections)
         {
            connections.put(id, agent);
         }

         agent.bind();
         
         return id;
      }

      /**
       * Terminate the connection with the given ID and unbind the agent.
       * 
       * @param    id
       *           The connection ID.
       * @param    running
       *           Flag indicating the Agent is already processing a command.
       */
      @Override
      void terminateConnection(String id, boolean running)
      {
         super.terminateConnection(id, running);
         
         Agent agent = null;
         synchronized (connections)
         {
            agent = connections.remove(id);
         }
         
         if (reset)
         {
            // reset the agent's context
            agent.resetContext(running ? InvocationModes.IMMEDIATELY : InvocationModes.AGENT_WAIT);
         }

         agent.deleteProcedures(id);
         
         poolAgent(agent, false);
      }
      
      /**
       * Determine if the agent must be unbound and deactivate procedure invoked, when there are no more
       * persistent procedures bound to it.
       * 
       * @param    id
       *           The connection ID.
       * @param    referent
       *           The persistent procedure.
       *           
       * @return   <code>true</code> if the deactivate can be invoked and the agent unbound.
       */
      @Override
      boolean deactivateAgentOnRemoveProcedure(String id, Object referent)
      {
         return false;
      }
   }
   
   /**
    * Class implementing the behavior for STATE-FREE and STATELESS operating modes.  In these
    * modes, no agent is bound to the connection when the connection is established.
    */
   static class UnboundPool
   extends AgentPool
   {
      /**
       * The reserved agent, for each connection.  This map is populated only when the agent becomes bound
       * to the connection.
       */
      private final Map<String, Agent> connections = new HashMap<String, Agent>();
      
      /** Flag indicating this is a STATE-FREE mode. */
      private final boolean free;
      
      /**
       * Build a new agent pool, for the specified appserver.
       * 
       * @param    appDef
       *           The appserver definition.
       * @param    free
       *           Flag indicating this is a STATE-FREE mode.
       */
      UnboundPool(ClassicAppserverDefinition appDef, boolean free)
      {
         super(appDef);
         this.free = free;
      }

      /**
       * Get the agent associated with the given connection id.  If no agent is yet bound to the connection
       * or to the persistent procedure, it extracts a new agent from the pool.
       * 
       * @param    id
       *           The connection ID.
       * @param    code
       *           The code of a persistent external procedure to which the agent is associated.
       *           May be <code>null</code>.
       * @param    agentId
       *           When non-negative, the agent ID which created the persistent procedure.
       *           
       * @return   See above.
       * 
       * @throws   NumberedException
       *           If there was no agent available.
       */
      @Override
      Agent acquireAgent(String id, String code, int agentId)
      throws NumberedException
      {
         Agent agent = null;
         
         // first check if we have a bound agent
         synchronized (connections)
         {
            agent = connections.get(id);
         }
         
         if (agent == null && code != null)
         {
            // no connection-bound agent, get the dedicated agent, if a procedure code is specified
            agent = getAgent(agentId);
            
            if (agent != null)
            {
               // validate the agent against the procedure code and the connection ID
               agent.validateRequest(id, code);
            }
            else
            {
               throw new NumberedException("The agent for the persistent procedure no longer exists.");
            }
         }
         
         if (agent == null)
         {
            // last resort, get one from main pool
            agent = getFromMainPool();
         }
         
         if (agent == null)
         {
            throw new NumberedException("No servers available");
         }

         return agent;
      }

      /**
       * Release the given agent and place it back in the pool, but only if is still unbound.
       * 
       * @param    id
       *           The connection ID.
       * @param    agent
       *           The agent which needs to be released.
       */
      @Override
      void releaseAgent(String id, Agent agent)
      {
         if (agent.terminated())
         {
            synchronized (connections)
            {
               connections.remove(id);
            }
            
            return;
         }

         if (agent.isBound())
         {
            if (!free)
            {
               // in non-free mode (Stateless), agents are bound only when a procedure is invoked PERSISTENT
               synchronized (connections)
               {
                  connections.put(id, agent);
               }
            }
            
            // otherwise, for State-free there is nothing to do, it remains bound to the persistent procedure, 
            // not connection
         }
         else
         {
            if (!free)
            {
               synchronized (connections)
               {
                  connections.remove(id);
               }
            }

            // if there was a bound agent, add it back to the main pool
            poolAgent(agent, false);
            
            // do not reset the context
         }
      }
      
      /**
       * Establish a new connection.
       * 
       * @return   The connection ID.
       */
      @Override
      String newConnection()
      {
         String id = nextId();
      
         synchronized (connections)
         {
            // no agent is initially bound to this pool
            connections.put(id, null);
         }
      
         return id;
      }

      /**
       * Terminate the connection with the given ID and unbind the agent.
       * 
       * @param    id
       *           The connection ID.
       * @param    running
       *           Flag indicating the Agent is already processing a command.
       */
      @Override
      void terminateConnection(String id, boolean running)
      {
         super.terminateConnection(id, running);

         Agent agent = null;
         synchronized (connections)
         {
            agent = connections.remove(id);
         }
         
         if (agent != null)
         {
            agent.deleteProcedures(id);

            // if there was a bound agent, add it back to the main pool
            poolAgent(agent, false);
         }
         
         // any agent still bound to a persistent procedure created on this connection will be released.
         
         Set<Agent> unbind = new HashSet<>();
         synchronized (persistentProcedures)
         {
            for (Agent a : persistentProcedures.values())
            {
               if (a.hasProcedures(id))
               {
                  unbind.add(a);
               }
            }
         }
         
         for (Agent a : unbind)
         {
            a.deleteProcedures(id);
         }
      }
      
      /**
       * Determine if the agent must be unbound and deactivate procedure invoked, when there are no more
       * persistent procedures bound to it.
       * 
       * @param    id
       *           The connection ID.
       * @param    referent
       *           The persistent procedure.
       *           
       * @return   <code>true</code> if the deactivate can be invoked and the agent unbound.
       */
      @Override
      boolean deactivateAgentOnRemoveProcedure(String id, Object referent)
      {
         Agent agent = null;
         synchronized (persistentProcedures)
         {
            agent = persistentProcedures.get(referent);
         }
         
         return agent != null && !agent.hasProcedures(id);
      }
   }
}