# AppServerLauncher.java

    /*
    ** Module   : AppServerLauncher.java
    ** Abstract : Class dedicated to launching appservers.
    **
    ** Copyright (c) 2013-2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------------Description----------------------------------------
    ** 001 CA  20130705 Created initial version.
    ** 002 CA  20140124 Mark the client to be launched batch mode. Redirect the standard output and
    **                  error streams of the client process to a file.
    ** 003 CA  20140206 Changed to launch the appserver using the spawner and scheduler infrastructure.
    **                  Moved logging support to use the logging provided by LogHelper.
    ** 004 MAG 20140310 When running on Windows OS we have to provide a password as well.
    ** 005 OM  20150108 Added properties to environment map of the spawned client process.
    ** 006 CA  20161027 When a new Agent needs to be started (if current agents are all busy), then
    **                  the Agent launch must be done from the server context.  This is solved by
    **                  scheduling a JobType.INSTANCE to be ran immediately, to spawn the Agent.
    ** 007 CA  20191211 When spawning a Java process programmatically, allow the caller to pass
    **                  environment options specific to that call (beside any jvmArgs configured in
    **                  the directory).
    ** 008 CA  20200110 The job for agent launching must have an unique name.
    **     CA  20221031 Measure and report the time elapsed to launch a new agent.
    **     CA  20230130 Added options to trim or restart the agents in an appserver.
    ** 009 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    ** 010 GBB 20240311 Log message for agent spawn succeess / error improved.
    ** 011 GBB 20240709 Pass params to ProcessClientSpawner constructor instead of method for immutability.
    ** 012 GBB 20241010 Duplicated method trimAgents removed (replaced by trim). Adding method terminateAgents.
    ** 013 GBB 20250116 Async launch rework.
    ** 014 CA  20250203 If 'getLaunchWorkerFutures' is not executed on a FWD server thread, then the job needs
    **                  to be posted to the FWD scheduler and ran from there.
    ** 015 AP  20250310 Added parameter for default timeout.
    ** 016 GBB 20250403 Support MSA server. Added terminateSessions.
    */
    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util;

    import java.util.*;
    import java.util.concurrent.*;
    import java.util.concurrent.atomic.*;
    import java.util.logging.*;

    import com.goldencode.p2j.admin.*;
    import com.goldencode.p2j.main.*;
    import com.goldencode.p2j.scheduler.*;
    import com.goldencode.p2j.security.SecurityManager;
    import com.goldencode.p2j.util.appserver.*;
    import com.goldencode.p2j.util.logging.*;

    /**
     * This is a launcher of appservers and is used in two ways:
     * <ul>
     *    <li>Automatically starts the appservers which have the <code>auto_start=true</code> in their
     *        configuration; this will start up to <code>initial_agents</code>.</li>
     *    <li>Allow starting of appservers remotely; if a server is started using the "-a appserver"
     *        argument, it will connect to the target P2J Server and call {@link AppServer#launch} to
     *        launch the specified appserver. If the appserver is already started, it will attempt to
     *        start an agent.</li>
     * </ul>
     *
     * There is one other way to start an appserver: by manually specifying the ID of an account
     * configured to start an appserver, when starting a P2J Client.  This will start only one Agent
     * (if non started).
     */
    public final class AppServerLauncher
    {
       /** Logger */
       private static final CentralLogger LOG = CentralLogger.get(AppServerLauncher.class.getName());

       /** The maximum number of continous failures to start an Agent, before giving up. */
       private static final int MAX_FAILURES = 10;

       /** The maximum amount of time to wait for an Agent to start. */
       private static final int LAUNCH_TIMEOUT = 120_000;

       /** Lock for adding / removing listeners for newly initialized agents. */
       private static final Object AGENT_LISTENERS_LOCK = new Object();

       /** Map of listeners for newly initialized agents per appserver. */
       private static final Map<String, Queue<InitAgentListener>> appserverAgentListeners = new HashMap<>();

       /** Executor with threads for spawning new agents. */
       private static final ExecutorService spawnAgentExecutor = Executors.newCachedThreadPool(new ThreadFactory()
       {
          /** Counter for unique thread numbers. */
          private final AtomicInteger threadCounter = new AtomicInteger(0);

          /**
           * Creates a new thread for the provided runnable and assigns a name to it that identifies it uniquely.
           *
           * @param    runnable
           *           The task to be executed by the thread.
           *
           * @return   The new thread.
           */
          @Override
          public Thread newThread(Runnable runnable)
          {
             Thread thread = new Thread(runnable, "launch-agent-" + threadCounter.incrementAndGet());
             thread.setDaemon(true);
             return thread;
          }
       });

       /** Map for agent trim timers to avoid being GCed */
       private static final Map<String, Timer> trimTimers = new ConcurrentHashMap<>();

       /** Map of spawner params per appserver */
       private static final Map<String, ProcessClientSpawner.Params> spawnerParams = new ConcurrentHashMap<>();

       /** Map of spawner params per appserver */
       private static final Set<String> initializedAppservers = new HashSet<>();

       /**
        * Launch the specified appserver. This includes starting the specified number of initial
        * agents. The agents will need to authenticate using the specified account.
        *
        * @param    appServer
        *           The appserver name to launch.
        *
        * @return   <code>true</code> if the appserver was launched.
        */
       public static boolean launch(String appServer)
       {
          MultiSessionAppserverDefinition msaDef = MultiSessionAppserverDefinition.get(appServer);
          if (msaDef != null)
          {
             return MultiSessionAppserverManager.getInstance().start(appServer);
          }

          ClassicAppserverDefinition appDef = ClassicAppserverDefinition.get(appServer);
          if (appDef == null)
          {
             return false;
          }

          if (!spawnerParams.containsKey(appServer))
          {
             addSpawnerParams(appDef);
          }

          synchronized (spawnerParams.get(appServer))
          {
             if (!initializedAppservers.contains(appServer))
             {
                return launchFirstAgents(appDef);
             }
          }
          return launchWorker(appDef);
       }

       /**
        * Restart all agents in the specified appserver.
        *
        * @param    appServer
        *           The appserver name to restart.
        */
       public static void restart(String appServer)
       {
          if (MultiSessionAppserverDefinition.get(appServer) != null)
          {
             MultiSessionAppserverManager.getInstance().restart(appServer);
             return;
          }
          AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
          if (pool == null)
          {
             LOG.log(Level.WARNING, "Attempting to restart agents in unknown appserver: " + appServer);
             return;
          }

          pool.restartAgents();
       }

       /**
        * Terminates the list of agents in the specified appserver.
        *
        * @param    appserver
        *           The appserver name to restart.
        * @param    agentIds
        *           The list of agent ids
        */
       public static void terminateAgents(String appserver, int[] agentIds)
       {
          if (MultiSessionAppserverDefinition.get(appserver) != null)
          {
             MultiSessionAppserverManager.getInstance().terminateAgents(appserver, agentIds);
             return;
          }
          AgentPool pool = AppServerManager.getAgentPoolForAppserver(appserver);
          if (pool == null)
          {
             LOG.log(Level.WARNING, "Attempting to trim agents in unknown appserver: " + appserver);
             return;
          }

          for (int agentId : agentIds)
          {
             LOG.log(Level.INFO, "Terminating agent with id %d in appserver %s.", agentId, appserver);
             pool.getAgent(agentId).terminate();
          }
       }

       /**
        * Terminates the selected sessions of the appserver and returns the refreshed list of appserver agents.
        *
        * @param    appserver
        *           The appserver name
        * @param    sessions
        *           The list of sessions to be terminated
        */
       public static void terminateSessions(String appserver, MultiSessionAgentSessionDef[] sessions)
       {
          MultiSessionAppserverManager.getInstance().terminateSessions(appserver, sessions);
       }

       /**
        * Trim the agents in the specified appserver.
        *
        * @param    appServer
        *           The appserver name to trim.
        */
       public static void trim(String appServer)
       {
          if (MultiSessionAppserverDefinition.get(appServer) != null)
          {
             MultiSessionAppserverManager.getInstance().trim(appServer);
             return;
          }
          AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
          if (pool == null)
          {
             LOG.log(Level.WARNING, "Attempting to trim agents in unknown appserver: " + appServer);
             return;
          }

          pool.trimAgents();
       }

       /**
        * When the appserver Agent is actually about to start listening for commands, it will notify
        * its launcher that it was started. Until then, the appserver Agent is presumed "not running".
        * <p>
        * Note that when an appserver is configured with <code>auto_start=false</code> and the agent
        * is started manually, no {@link #appserverAgentListeners} will be available. Listeners are used only
        * when the appserver is started automatically or when the server receives the "-p appserver" argument.
        *
        * @param    appServer
        *           The name of the appserver with which the Agent is associated.
        */
       static void notifyLaunch(String appServer)
       {
          synchronized (AGENT_LISTENERS_LOCK)
          {
             Queue<InitAgentListener> agentListeners = appserverAgentListeners.get(appServer);
             while (agentListeners.size() > 0)
             {
                InitAgentListener agentListener = agentListeners.poll();
                if (agentListener != null && !agentListener.isCancelled())
                {
                   agentListener.onNewAgent();
                   return;
                }
             }
          }
       }

       /**
        * Terminate the given appserver.
        *
        * @param    appServer
        *           The name of the appserver to terminate.
        */
       public static void terminate(String appServer)
       {
          AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
          pool.terminateAll();
       }

       /**
        * Notify that the given appserver has been terminated (there are no more agents for it and
        * the pool is shutting down).
        *
        * @param    appServer
        *           The name of the appserver being terminated.
        */
       static void terminated(String appServer)
       {
          synchronized (AGENT_LISTENERS_LOCK)
          {
             Queue<InitAgentListener> initListeners = appserverAgentListeners.remove(appServer);
             if (initListeners != null && initListeners.size() > 0)
             {
                while (initListeners.size() > 0)
                {
                   initListeners.poll().cancel(true);
                }
             }
          }

          spawnerParams.remove(appServer);
          initializedAppservers.remove(appServer);

          Timer trimTimer = trimTimers.remove(appServer);
          if (trimTimer != null)
          {
             trimTimer.cancel();
          }
       }

       /**
        * Initializes the appserver by starting an <code>initial_agents</code> Agents.  An Agent startup will
        * fail if it takes more than {@value #LAUNCH_TIMEOUT} seconds.  Also, the Agent startup will not
        * continue if it failed more than {@link #MAX_FAILURES} times in a row.
        * <p>
        * If first call and at least one agent has been started, it will start a thread responsible
        * of sending the
        *
        * @param    appDef
        *           The appserver definition.
        *
        * @return   <code>true</code> if the appserver or agent was started successfully.
        */
       private static boolean launchFirstAgents(ClassicAppserverDefinition appDef)
       {
          long startTime = System.nanoTime();
          boolean success = true;
          String appServer = appDef.getAppserverName();

          final AtomicInteger failures = new AtomicInteger(0);
          int initialAgents = appDef.getInitialAgents();
          int agentsToLaunchCount = initialAgents;

          while (true)
          {
             // protect this loop so it is finite; if agent startup failed 10 times in a row, abend the process

             List<Tuple<Future<Boolean>, CompletableFuture<Boolean>>> allLaunchFutures = new ArrayList<>();

             // start all async launches
             for (int launchAttempt = 0; launchAttempt < agentsToLaunchCount; launchAttempt++)
             {
                Tuple<Future<Boolean>, CompletableFuture<Boolean>> futures = getLaunchWorkerFutures(appServer);
                allLaunchFutures.add(futures);
             }

             // wait for all launch results
             for (Tuple<Future<Boolean>, CompletableFuture<Boolean>> futuresTuple : allLaunchFutures)
             {
                if (!handleLaunchResult(appDef, futuresTuple, startTime))
                {
                   failures.incrementAndGet();
                }
             }

             AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);
             int initializedAgents = pool == null ? 0 : pool.size();

             if (initializedAgents >= initialAgents)
             {
                break;
             }
             if (failures.get() >= MAX_FAILURES)
             {
                success = false;
                break;
             }

             int remainingAgentsToLaunch = initialAgents - initializedAgents;
             int remainingAttempts = MAX_FAILURES - failures.get();
             agentsToLaunchCount = Math.min(remainingAttempts, remainingAgentsToLaunch);
          }

          AgentPool pool = AppServerManager.getAgentPoolForAppserver(appServer);

          if (pool != null && pool.size() > 0)
          {
             initializedAppservers.add(appServer);
          }

          if (!success && LOG.isLoggable(Level.WARNING))
          {
             if (pool == null)
             {
                LOG.log(Level.SEVERE, "Could not start appserver '%s'.", appServer);
             }
             else
             {
                LOG.log(Level.WARNING,
                        "Only %d out of %d were started for appserver %s",
                        pool.size(),
                        initialAgents,
                        appServer);
             }
          }

          return success;
       }

       /**
        * Create launch worker Future.
        *
        * @param    appServer
        *           The appserver name.
        *
        * @return   Tuple with first value the spawn Future and
        *           second value the initialization listener CompletableFuture.
        */
       private static Tuple<Future<Boolean>, CompletableFuture<Boolean>> getLaunchWorkerFutures(String appServer)
       {
          if (!spawnerParams.containsKey(appServer))
          {
             return null;
          }
          boolean srvAccount = SecurityManager.getInstance().isServerAccount();
          ProcessClientSpawner spawner = srvAccount ? new ProcessClientSpawner(spawnerParams.get(appServer))
                                                    : null;

          InitAgentListener initAgentFuture = new InitAgentListener();
          Future<Boolean> spawnFuture = spawnAgentExecutor.submit(() ->
          {
             String jobName = "launch-agent-" + appServer + "-" + UUID.randomUUID().toString();
             Thread.currentThread().setName(jobName);

             synchronized (AGENT_LISTENERS_LOCK)
             {
                if (!appserverAgentListeners.containsKey(appServer))
                {
                   appserverAgentListeners.put(appServer, new LinkedBlockingQueue<>());
                }

                Queue<InitAgentListener> newAgentListeners = appserverAgentListeners.get(appServer);
                newAgentListeners.add(initAgentFuture);
             }

             if (!srvAccount)
             {
                Runnable launchJob = () ->
                {
                   new ProcessClientSpawner(spawnerParams.get(appServer)).spawn();
                };
                Job job = new Job(jobName, launchJob, true, JobMode.NOW);
                if (!Scheduler.addJob(job))
                {
                   throw new IllegalStateException("A job with for " + jobName + " is already configured!");
                }
                Scheduler.scheduleJob(jobName, null);
             }
             else
             {
                spawner.spawn();
             }

             return initAgentFuture.get();
          });

          return new Tuple<>(spawnFuture, initAgentFuture);
       }

       /**
        * Worker for Agent launching. After starting the Agent via the {@link ProcessClientSpawner#spawn()},
        * it will wait for {@link #LAUNCH_TIMEOUT} seconds for the Agent to call {@link #notifyLaunch}.
        * <p>
        * After the timeout or when it receives the launch notification, it will check the for the status of
        * the launch. The launch notification may come from a different Agent than the one currently starting.
        * This is not a problem, the party starting the Agent doesn't care which agent started, it will only
        * need to know if an Agent was started; other parties waiting for Agent startup will eventually pick up
        * the launch notification, if the Agent was started.
        *
        * @param   appDef
        *          The definition for the appserver.
        *
        * @return   <code>true</code> if the Agent was started successfully.
        */
       private static boolean launchWorker(ClassicAppserverDefinition appDef)
       {
          long startTime = System.nanoTime();

          String appServer = appDef.getAppserverName();

          Tuple<Future<Boolean>, CompletableFuture<Boolean>> futures = getLaunchWorkerFutures(appServer);

          return handleLaunchResult(appDef, futures, startTime);
       }

       /**
        * Waits for results from the launch of the new agent and handles the response. In case of the launch
        * Future times out, we're confirming the agent init listener has not received a response in the time
        * after the timeout by synchronizing {@link #AGENT_LISTENERS_LOCK} and getting its response.
        *
        * @param   appDef
        *          The appserver definition.
        * @param   futures
        *          The Future objects associated with the agent launch.
        * @param   startTime
        *          The start time of the appserver agent launch.
        *
        * @return  <code>true</code> if the agent was successfully initialized, <code>false</code> otherwise.
        */
       private static boolean handleLaunchResult(ClassicAppserverDefinition appDef,
                                                 Tuple<Future<Boolean>, CompletableFuture<Boolean>> futures,
                                                 long startTime)
       {
          String appServer = appDef.getAppserverName();

          if (futures == null)
          {
             return false;
          }

          Future<Boolean> launchFuture = futures.getKey();
          CompletableFuture<Boolean> initAgentFuture = futures.getValue();

          try
          {
             // timeout is on the spawn Future to be able to catch exceptions
             launchFuture.get(LAUNCH_TIMEOUT, TimeUnit.MILLISECONDS);
          }
          catch (Throwable t)
          {
             LOG.log(Level.FINE, "Exception in handleLaunchResult:", t);
          }

          boolean isSuccess;
          synchronized (AGENT_LISTENERS_LOCK)
          {
             // checking if it has been completed after the main Future timeout / throwable
             isSuccess = initAgentFuture.getNow(false);
             initAgentFuture.cancel(true);
          }

          long elapsed = System.nanoTime() - startTime;
          long millis = elapsed / 1000000;

          if (isSuccess)
          {
             LOG.log(Level.INFO, "Agent for appserver '%s' was started successfully (%dms)!", appServer, millis);
             startAutoTrimTimer(appDef);
          }
          else
          {
             LOG.log(Level.WARNING, "Agent for appserver '%s' could not be started (%dms)!", appServer, millis);
          }

          return isSuccess;
       }

       /**
        * Create, start and add to {@link #trimTimers} timer to trim down appserver agents.
        *
        * @param   appDef
        *          The definition for the appserver.
        */
       private static void startAutoTrimTimer(ClassicAppserverDefinition appDef)
       {
          String appServer = appDef.getAppserverName();

          synchronized (trimTimers)
          {
             if (trimTimers.containsKey(appServer))
             {
                return;
             }
             Timer timer = new Timer(appServer + " agent trimming thread.", true);
             trimTimers.put(appServer, timer);

             TimerTask tt = new TimerTask()
             {
                public void run()
                {
                   trim(appServer);
                }
             };
             long timeout = appDef.getAutoTrimTimeout() * 1000;
             timer.scheduleAtFixedRate(tt, timeout, timeout);
          }
       }

       /**
        * Adds a spawner for the appserver.
        *
        * @param   appDef
        *          The appserver definition.
        */
       private static void addSpawnerParams(AppserverDefinition appDef)
       {
          String appServer = appDef.getAppserverName();
          SecurityManager secMgr = SecurityManager.getInstance();

          // find the user of this appserver
          String[] subjects = secMgr.getAccountsForAppserver(appServer);
          if (subjects.length == 0)
          {
             final String msg = "Appserver '%s' has no associated user.";
             throw new IllegalStateException(String.format(msg, appServer));
          }
          else if (subjects.length > 1)
          {
             final String msg = "Appserver '%s' is associated with more than one user: %s";
             throw new IllegalStateException(String.format(msg,
                                                           appServer,
                                                           Arrays.asList(subjects)));
          }

          String fwdAccount = subjects[0];
          String systemUser = secMgr.getSystemUser(fwdAccount);
          String systemPass = secMgr.getSystemPassword(fwdAccount);
          Map<String, String> envMap = appDef.getProperties();

          int spawnerLaunchDefaultTimeoutSeconds = appDef instanceof ClassicAppserverDefinition ?
             ((ClassicAppserverDefinition) appDef).getRequestTimeout() :
             ((MultiSessionAppserverDefinition) appDef).getAgentListenerTimeout() / 1000;

          spawnerParams.putIfAbsent(appServer,
                                    new ProcessClientSpawner.Params(fwdAccount, systemUser, systemPass, envMap,
                                                                    spawnerLaunchDefaultTimeoutSeconds));
       }

       /**
        * CompletableFuture that completes on receiving an event for a new agent being initialized.
        */
       private static class InitAgentListener
       extends CompletableFuture<Boolean>
       {
          /**
           * Method called to indicate a new agent has been initialized.
           */
          void onNewAgent()
          {
             if (!isCancelled())
             {
                complete(true);
             }
          }
       }
    }