AppServerLauncher.java
/*
** Module : AppServerLauncher.java
** Abstract : Class dedicated to launching appservers.
**
** Copyright (c) 2013-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description----------------------------------------
** 001 CA 20130705 Created initial version.
** 002 CA 20140124 Mark the client to be launched batch mode. Redirect the standard output and
** error streams of the client process to a file.
** 003 CA 20140206 Changed to launch the appserver using the spawner and scheduler infrastructure.
** Moved logging support to use the logging provided by LogHelper.
** 004 MAG 20140310 When running on Windows OS we have to provide a password as well.
** 005 OM 20150108 Added properties to environment map of the spawned client process.
** 006 CA 20161027 When a new Agent needs to be started (if current agents are all busy), then
** the Agent launch must be done from the server context. This is solved by
** scheduling a JobType.INSTANCE to be ran immediately, to spawn the Agent.
** 007 CA 20191211 When spawning a Java process programmatically, allow the caller to pass
** environment options specific to that call (beside any jvmArgs configured in
** the directory).
** 008 CA 20200110 The job for agent launching must have an unique name.
** CA 20221031 Measure and report the time elapsed to launch a new agent.
** CA 20230130 Added options to trim or restart the agents in an appserver.
** 009 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 010 GBB 20240311 Log message for agent spawn succeess / error improved.
** 011 GBB 20240709 Pass params to ProcessClientSpawner constructor instead of method for immutability.
** 012 GBB 20241010 Duplicated method trimAgents removed (replaced by trim). Adding method terminateAgents.
** 013 GBB 20250116 Async launch rework.
** 014 CA 20250203 If 'getLaunchWorkerFutures' is not executed on a FWD server thread, then the job needs
** to be posted to the FWD scheduler and ran from there.
** 015 AP 20250310 Added parameter for default timeout.
** 016 GBB 20250403 Support MSA server. Added terminateSessions.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.util;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.logging.*;
import com.goldencode.p2j.admin.*;
import com.goldencode.p2j.main.*;
import com.goldencode.p2j.scheduler.*;
import com.goldencode.p2j.security.SecurityManager;
import com.goldencode.p2j.util.appserver.*;
import com.goldencode.p2j.util.logging.*;
/**
* This is a launcher of appservers and is used in two ways:
* <ul>
* <li>Automatically starts the appservers which have the <code>auto_start=true</code> in their
* configuration; this will start up to <code>initial_agents</code>.</li>
* <li>Allow starting of appservers remotely; if a server is started using the "-a appserver"
* argument, it will connect to the target P2J Server and call {@link AppServer#launch} to
* launch the specified appserver. If the appserver is already started, it will attempt to
* start an agent.</li>
* </ul>
*
* There is one other way to start an appserver: by manually specifying the ID of an account
* configured to start an appserver, when starting a P2J Client. This will start only one Agent
* (if non started).
*/
public final class AppServerLauncher
{
/** Logger */
private static final CentralLogger LOG = CentralLogger.get(AppServerLauncher.class.getName());
/** The maximum number of continous failures to start an Agent, before giving up. */
private static final int MAX_FAILURES = 10;
/** The maximum amount of time to wait for an Agent to start. */
private static final int LAUNCH_TIMEOUT = 120_000;
/** Lock for adding / removing listeners for newly initialized agents. */
private static final Object AGENT_LISTENERS_LOCK = new Object();
/** Map of listeners for newly initialized agents per appserver. */
private static final Map<String, Queue<InitAgentListener>> appserverAgentListeners = new HashMap<>();
/** Executor with threads for spawning new agents. */
private static final ExecutorService spawnAgentExecutor = Executors.newCachedThreadPool(new ThreadFactory()
{
/** Counter for unique thread numbers. */
private final AtomicInteger threadCounter = new AtomicInteger(0);
/**
* Creates a new thread for the provided runnable and assigns a name to it that identifies it uniquely.
*
* @param runnable
* The task to be executed by the thread.
*
* @return The new thread.
*/
@Override
public Thread newThread(Runnable runnable)
{
Thread thread = new Thread(runnable, "launch-agent-" + threadCounter.incrementAndGet());
thread.setDaemon(true);
return thread;
}
});
/** Map for agent trim timers to avoid being GCed */
private static final Map<String, Timer> trimTimers = new ConcurrentHashMap<>();
/** Map of spawner params per appserver */
private static final Map<String, ProcessClientSpawner.Params> spawnerParams = new ConcurrentHashMap<>();
/** Map of spawner params per appserver */
private static final Set<String> initializedAppservers = new HashSet<>();
/**
* Launch the specified appserver. This includes starting the specified number of initial
* agents. The agents will need to authenticate using the specified account.
*
* @param appServer
* The appserver name to launch.
*
* @return <code>true</code> if the appserver was launched.
*/
public static boolean launch(String appServer)
{
MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appServer);
if (msaDef != null)
{
return MultiSessionAppserverManager.getInstance().start(appServer);
}
ClassicAppserverDefinition appDef = ClassicAppserverDefinition.get(appServer);
if (appDef == null)
{
return false;
}
if (!spawnerParams.containsKey(appServer))
{
addSpawnerParams(appDef);
}
synchronized (spawnerParams.get(appServer))
{
if (!initializedAppservers.contains(appServer))
{
return launchFirstAgents(appDef);
}
}
return launchWorker(appDef);
}
/**
* Restart all agents in the specified appserver.
*
* @param appServer
* The appserver name to restart.
*/
public static void restart(String appServer)
{
if (MultiSessionAppserverDefinition.get(appServer) != null)
{
MultiSessionAppserverManager.getInstance().restart(appServer);
return;
}
AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
if (pool == null)
{
LOG.log(Level.WARNING, "Attempting to restart agents in unknown appserver: " + appServer);
return;
}
pool.restartAgents();
}
/**
* Terminates the list of agents in the specified appserver.
*
* @param appserver
* The appserver name to restart.
* @param agentIds
* The list of agent ids
*/
public static void terminateAgents(String appserver, int[] agentIds)
{
if (MultiSessionAppserverDefinition.get(appserver) != null)
{
MultiSessionAppserverManager.getInstance().terminateAgents(appserver, agentIds);
return;
}
AgentPool pool = AppServerManager.getAgentPoolForAppserver(appserver);
if (pool == null)
{
LOG.log(Level.WARNING, "Attempting to trim agents in unknown appserver: " + appserver);
return;
}
for (int agentId : agentIds)
{
LOG.log(Level.INFO, "Terminating agent with id %d in appserver %s.", agentId, appserver);
pool.getAgent(agentId).terminate();
}
}
/**
* Terminates the selected sessions of the appserver and returns the refreshed list of appserver agents.
*
* @param appserver
* The appserver name
* @param sessions
* The list of sessions to be terminated
*/
public static void terminateSessions(String appserver, MultiSessionAgentSessionDef[] sessions)
{
MultiSessionAppserverManager.getInstance().terminateSessions(appserver, sessions);
}
/**
* Trim the agents in the specified appserver.
*
* @param appServer
* The appserver name to trim.
*/
public static void trim(String appServer)
{
if (MultiSessionAppserverDefinition.get(appServer) != null)
{
MultiSessionAppserverManager.getInstance().trim(appServer);
return;
}
AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
if (pool == null)
{
LOG.log(Level.WARNING, "Attempting to trim agents in unknown appserver: " + appServer);
return;
}
pool.trimAgents();
}
/**
* When the appserver Agent is actually about to start listening for commands, it will notify
* its launcher that it was started. Until then, the appserver Agent is presumed "not running".
* <p>
* Note that when an appserver is configured with <code>auto_start=false</code> and the agent
* is started manually, no {@link #appserverAgentListeners} will be available. Listeners are used only
* when the appserver is started automatically or when the server receives the "-p appserver" argument.
*
* @param appServer
* The name of the appserver with which the Agent is associated.
*/
static void notifyLaunch(String appServer)
{
synchronized (AGENT_LISTENERS_LOCK)
{
Queue<InitAgentListener> agentListeners = appserverAgentListeners.get(appServer);
while (agentListeners.size() > 0)
{
InitAgentListener agentListener = agentListeners.poll();
if (agentListener != null && !agentListener.isCancelled())
{
agentListener.onNewAgent();
return;
}
}
}
}
/**
* Terminate the given appserver.
*
* @param appServer
* The name of the appserver to terminate.
*/
public static void terminate(String appServer)
{
AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
pool.terminateAll();
}
/**
* Notify that the given appserver has been terminated (there are no more agents for it and
* the pool is shutting down).
*
* @param appServer
* The name of the appserver being terminated.
*/
static void terminated(String appServer)
{
synchronized (AGENT_LISTENERS_LOCK)
{
Queue<InitAgentListener> initListeners = appserverAgentListeners.remove(appServer);
if (initListeners != null && initListeners.size() > 0)
{
while (initListeners.size() > 0)
{
initListeners.poll().cancel(true);
}
}
}
spawnerParams.remove(appServer);
initializedAppservers.remove(appServer);
Timer trimTimer = trimTimers.remove(appServer);
if (trimTimer != null)
{
trimTimer.cancel();
}
}
/**
* Initializes the appserver by starting an <code>initial_agents</code> Agents. An Agent startup will
* fail if it takes more than {@value #LAUNCH_TIMEOUT} seconds. Also, the Agent startup will not
* continue if it failed more than {@link #MAX_FAILURES} times in a row.
* <p>
* If first call and at least one agent has been started, it will start a thread responsible
* of sending the
*
* @param appDef
* The appserver definition.
*
* @return <code>true</code> if the appserver or agent was started successfully.
*/
private static boolean launchFirstAgents(ClassicAppserverDefinition appDef)
{
long startTime = System.nanoTime();
boolean success = true;
String appServer = appDef.getAppserverName();
final AtomicInteger failures = new AtomicInteger(0);
int initialAgents = appDef.getInitialAgents();
int agentsToLaunchCount = initialAgents;
while (true)
{
// protect this loop so it is finite; if agent startup failed 10 times in a row, abend the process
List<Tuple<Future<Boolean>, CompletableFuture<Boolean>>> allLaunchFutures = new ArrayList<>();
// start all async launches
for (int launchAttempt = 0; launchAttempt < agentsToLaunchCount; launchAttempt++)
{
Tuple<Future<Boolean>, CompletableFuture<Boolean>> futures = getLaunchWorkerFutures(appServer);
allLaunchFutures.add(futures);
}
// wait for all launch results
for (Tuple<Future<Boolean>, CompletableFuture<Boolean>> futuresTuple : allLaunchFutures)
{
if (!handleLaunchResult(appDef, futuresTuple, startTime))
{
failures.incrementAndGet();
}
}
AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
int initializedAgents = pool == null ? 0 : pool.size();
if (initializedAgents >= initialAgents)
{
break;
}
if (failures.get() >= MAX_FAILURES)
{
success = false;
break;
}
int remainingAgentsToLaunch = initialAgents - initializedAgents;
int remainingAttempts = MAX_FAILURES - failures.get();
agentsToLaunchCount = Math.min(remainingAttempts, remainingAgentsToLaunch);
}
AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
if (pool != null && pool.size() > 0)
{
initializedAppservers.add(appServer);
}
if (!success && LOG.isLoggable(Level.WARNING))
{
if (pool == null)
{
LOG.log(Level.SEVERE, "Could not start appserver '%s'.", appServer);
}
else
{
LOG.log(Level.WARNING,
"Only %d out of %d were started for appserver %s",
pool.size(),
initialAgents,
appServer);
}
}
return success;
}
/**
* Create launch worker Future.
*
* @param appServer
* The appserver name.
*
* @return Tuple with first value the spawn Future and
* second value the initialization listener CompletableFuture.
*/
private static Tuple<Future<Boolean>, CompletableFuture<Boolean>> getLaunchWorkerFutures(String appServer)
{
if (!spawnerParams.containsKey(appServer))
{
return null;
}
boolean srvAccount = SecurityManager.getInstance().isServerAccount();
ProcessClientSpawner spawner = srvAccount ? new ProcessClientSpawner(spawnerParams.get(appServer))
: null;
InitAgentListener initAgentFuture = new InitAgentListener();
Future<Boolean> spawnFuture = spawnAgentExecutor.submit(() ->
{
String jobName = "launch-agent-" + appServer + "-" + UUID.randomUUID().toString();
Thread.currentThread().setName(jobName);
synchronized (AGENT_LISTENERS_LOCK)
{
if (!appserverAgentListeners.containsKey(appServer))
{
appserverAgentListeners.put(appServer, new LinkedBlockingQueue<>());
}
Queue<InitAgentListener> newAgentListeners = appserverAgentListeners.get(appServer);
newAgentListeners.add(initAgentFuture);
}
if (!srvAccount)
{
Runnable launchJob = () ->
{
new ProcessClientSpawner(spawnerParams.get(appServer)).spawn();
};
Job job = new Job(jobName, launchJob, true, JobMode.NOW);
if (!Scheduler.addJob(job))
{
throw new IllegalStateException("A job with for " + jobName + " is already configured!");
}
Scheduler.scheduleJob(jobName, null);
}
else
{
spawner.spawn();
}
return initAgentFuture.get();
});
return new Tuple<>(spawnFuture, initAgentFuture);
}
/**
* Worker for Agent launching. After starting the Agent via the {@link ProcessClientSpawner#spawn()},
* it will wait for {@link #LAUNCH_TIMEOUT} seconds for the Agent to call {@link #notifyLaunch}.
* <p>
* After the timeout or when it receives the launch notification, it will check the for the status of
* the launch. The launch notification may come from a different Agent than the one currently starting.
* This is not a problem, the party starting the Agent doesn't care which agent started, it will only
* need to know if an Agent was started; other parties waiting for Agent startup will eventually pick up
* the launch notification, if the Agent was started.
*
* @param appDef
* The definition for the appserver.
*
* @return <code>true</code> if the Agent was started successfully.
*/
private static boolean launchWorker(ClassicAppserverDefinition appDef)
{
long startTime = System.nanoTime();
String appServer = appDef.getAppserverName();
Tuple<Future<Boolean>, CompletableFuture<Boolean>> futures = getLaunchWorkerFutures(appServer);
return handleLaunchResult(appDef, futures, startTime);
}
/**
* Waits for results from the launch of the new agent and handles the response. In case of the launch
* Future times out, we're confirming the agent init listener has not received a response in the time
* after the timeout by synchronizing {@link #AGENT_LISTENERS_LOCK} and getting its response.
*
* @param appDef
* The appserver definition.
* @param futures
* The Future objects associated with the agent launch.
* @param startTime
* The start time of the appserver agent launch.
*
* @return <code>true</code> if the agent was successfully initialized, <code>false</code> otherwise.
*/
private static boolean handleLaunchResult(ClassicAppserverDefinition appDef,
Tuple<Future<Boolean>, CompletableFuture<Boolean>> futures,
long startTime)
{
String appServer = appDef.getAppserverName();
if (futures == null)
{
return false;
}
Future<Boolean> launchFuture = futures.getKey();
CompletableFuture<Boolean> initAgentFuture = futures.getValue();
try
{
// timeout is on the spawn Future to be able to catch exceptions
launchFuture.get(LAUNCH_TIMEOUT, TimeUnit.MILLISECONDS);
}
catch (Throwable t)
{
LOG.log(Level.FINE, "Exception in handleLaunchResult:", t);
}
boolean isSuccess;
synchronized (AGENT_LISTENERS_LOCK)
{
// checking if it has been completed after the main Future timeout / throwable
isSuccess = initAgentFuture.getNow(false);
initAgentFuture.cancel(true);
}
long elapsed = System.nanoTime() - startTime;
long millis = elapsed / 1000000;
if (isSuccess)
{
LOG.log(Level.INFO, "Agent for appserver '%s' was started successfully (%dms)!", appServer, millis);
startAutoTrimTimer(appDef);
}
else
{
LOG.log(Level.WARNING, "Agent for appserver '%s' could not be started (%dms)!", appServer, millis);
}
return isSuccess;
}
/**
* Create, start and add to {@link #trimTimers} timer to trim down appserver agents.
*
* @param appDef
* The definition for the appserver.
*/
private static void startAutoTrimTimer(ClassicAppserverDefinition appDef)
{
String appServer = appDef.getAppserverName();
synchronized (trimTimers)
{
if (trimTimers.containsKey(appServer))
{
return;
}
Timer timer = new Timer(appServer + " agent trimming thread.", true);
trimTimers.put(appServer, timer);
TimerTask tt = new TimerTask()
{
public void run()
{
trim(appServer);
}
};
long timeout = appDef.getAutoTrimTimeout() * 1000;
timer.scheduleAtFixedRate(tt, timeout, timeout);
}
}
/**
* Adds a spawner for the appserver.
*
* @param appDef
* The appserver definition.
*/
private static void addSpawnerParams(AppserverDefinition appDef)
{
String appServer = appDef.getAppserverName();
SecurityManager secMgr = SecurityManager.getInstance();
// find the user of this appserver
String[] subjects = secMgr.getAccountsForAppserver(appServer);
if (subjects.length == 0)
{
final String msg = "Appserver '%s' has no associated user.";
throw new IllegalStateException(String.format(msg, appServer));
}
else if (subjects.length > 1)
{
final String msg = "Appserver '%s' is associated with more than one user: %s";
throw new IllegalStateException(String.format(msg,
appServer,
Arrays.asList(subjects)));
}
String fwdAccount = subjects[0];
String systemUser = secMgr.getSystemUser(fwdAccount);
String systemPass = secMgr.getSystemPassword(fwdAccount);
Map<String, String> envMap = appDef.getProperties();
int spawnerLaunchDefaultTimeoutSeconds = appDef instanceof ClassicAppserverDefinition ?
((ClassicAppserverDefinition) appDef).getRequestTimeout() :
((MultiSessionAppserverDefinition) appDef).getAgentListenerTimeout() / 1000;
spawnerParams.putIfAbsent(appServer,
new ProcessClientSpawner.Params(fwdAccount, systemUser, systemPass, envMap,
spawnerLaunchDefaultTimeoutSeconds));
}
/**
* CompletableFuture that completes on receiving an event for a new agent being initialized.
*/
private static class InitAgentListener
extends CompletableFuture<Boolean>
{
/**
* Method called to indicate a new agent has been initialized.
*/
void onNewAgent()
{
if (!isCancelled())
{
complete(true);
}
}
}
}