/*
** Module   : WebService.java
** Abstract : A web service test.
**
** Copyright (c) 2021-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description----------------------------------------
** 001 CA  20210827 Created initial version.
** 002 CA  20210902 Allow default HTTP methods (POST for SOAP and GET for REST).
**                  Fixed service calculation if it contains the '://' string.  Also, the 'service' was made
**                  optional.
** 002 CA  20210906 The content for description and failure is wrapped only for web tests. 
** 003 CA  20220107 The default Content-type is set only if not explicitly defined at the test.
** 004 CA  20220118 Measure the web service's exact execution time, excluding the harness overhead to execute 
**                  the tests on this response or building the request.
**                  Fixed response body encoding, read the bytes not the text.
** 005 CA  20220622 Fixed a bug in processing the 'service' attribute - it was not replacing variables if the
**                  service was not a full URL.
**     TJD 20220803 Updated for Java 11
** 006 CA  20240306 Added flag to omit the response body in the results (as for performance testing, the body
**                  may be huge and also not needed).
** 007 CA  20240725 Added cookie support for request and response.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You should have received a copy of the GNU Affero General Public License
** along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package com.goldencode.harness.test;

import java.io.*;
import java.security.*;
import java.security.cert.*;
import java.util.*;
import java.util.regex.*;

import javax.net.ssl.*;

import org.apache.http.*;
import org.apache.http.client.*;
import org.apache.http.client.config.*;
import org.apache.http.client.methods.*;
import org.apache.http.config.*;
import org.apache.http.conn.*;
import org.apache.http.conn.socket.*;
import org.apache.http.conn.ssl.*;
import org.apache.http.impl.client.*;
import org.apache.http.impl.conn.*;
import org.apache.http.protocol.*;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;

import com.goldencode.harness.*;

/**
 * A generic configuration for a web service test.
 */
public abstract class WebService
extends Report
implements Testable
{
   /** The actual HTTP response body. */
   protected String responseBody;
   
   /** 
    * The service to execute.  Can contain variable references, which will be substituted.  If the result 
    * does not contain <code>://</code>, the configured endpoint will be prepended.
    */
   private String service = null;
   
   /** The HTTP request method. */
   private String method = null;
   
   /** The query parameters to add. */
   private Map<String, String> queryParameters = new LinkedHashMap<>();
   
   /** The request headers. */
   private Map<String, String> headers = new LinkedHashMap<>();
   
   /** The request cookies. */
   private Map<String, String> cookies = new LinkedHashMap<>();
   
   /** The HTTP request body configuration. */
   private HttpRequestBody requestBody;
   
   /** The tests to apply on the HTTP response. */
   private HttpResponseTest response;

   /** The client to execute the HTTP request. */
   private ThreadLocal<CloseableHttpClient> client = new ThreadLocal<CloseableHttpClient>()
   {
      protected CloseableHttpClient initialValue()
      {
         return createHttpClient(null, 0);
      };
   };
   
   /** The variable pool. */
   private ResourcePool pool;

   /** The web test configuration (endpoint, authorization, etc). */
   private WebSessionSettings cfg;

   /** The configuration for the web service authorization. */
   private WebAuthorization authorization;

   /**
    * Create a new test.
    * 
    * @param    name
    *           The short name of this test type.
    * @param    cfg
    *           The web test configuration (endpoint, authorization, etc).
    * @param    rc
    *           Object in which to store the details of non-testing failures.
    * @param    pool
    *           The variable pool.
    */
   public WebService(String name, WebSessionSettings cfg, FailureNotification rc, ResourcePool pool)
   {
      super(name, null);
      
      this.pool = pool;
      this.cfg = cfg;
   }
   
   /**
    * Pretty print the specified body.
    * 
    * @param    body
    *           The body.
    *           
    * @return   The formatted body.
    */
   protected abstract String prettyPrint(String body);
   
   /**
    * Get the default content-type for the request.
    * 
    * @return   The default content-type.
    */
   protected abstract String getDefaultContentType();
   
   /**
    * Prepare the returned HTTP response.
    * 
    * @return   <code>true</code> if the response could be prepared.
    */
   protected abstract boolean prepareResponse();
   
   /**
    * Release any resources allocated for the HTTP response.
    */
   protected abstract void releaseResponse();
   
   /**
    * Check if the path exists in the body.
    * 
    * @param    path
    *           The path to check.
    *           
    * @return   <code>true</code> if the path exists.
    */
   protected abstract Object checkPathExists(String path);
   
   /**
    * Check the body path for an exact match.
    * 
    * @param    path
    *           The path to check.
    * @param    value
    *           The expected value.
    *           
    * @return   The value, or <code>null</code> if it does not match.
    */
   protected abstract Object checkPathValueExact(String path, String value);
   
   /**
    * Check the body path for a pattern match.
    * 
    * @param    path
    *           The path to check.
    * @param    pattern
    *           The pattern to match.
    *           
    * @return   The value, or <code>null</code> if it does not match.
    */
   protected abstract Object checkPathValueRegex(String path, Pattern pattern);

   /**
    * Match the body against the template.
    * <p>
    * All included paths will be retained from the body first, and after that the excluded paths will be 
    * removed.
    * <p>
    * Whatever paths remain must match exactly with the template.
    * 
    * @param    templateBody
    *           The body template.
    * @param    includes
    *           The list of included paths.
    * @param    excludes
    *           The list of excluded paths.
    *           
    * @return   The failure list, or <code>null</code> if it matches.
    */
   protected abstract String checkBodyMatch(String templateBody, List<String> includes, List<String> excludes);
   
   /**
    * Replace the given path in the body with the specified value.
    * 
    * @param    body
    *           The body.
    * @param    path
    *           The path to replace.
    * @param    val
    *           The value to use.
    *           
    * @return   The processed body, or <code>null</code> in case of failure.
    */
   protected abstract String replacePath(String body, String path, String val);

   /**
    * Do not allow externals to set the start/end time, this will be set when actually invoking the web
    * service.
    * 
    * @param    start
    *           The number of milliseconds.
    */
   @Override
   public final void setStartTime(long start)
   {
      // no-op
   }
   
   /**
    * Do not allow externals to set the start/end time, this will be set when actually invoking the web
    * service.
    * 
    * @param    end
    *           The number of milliseconds.
    */
   @Override
   public final void setEndTime(long end)
   {
      // no-op
   }
   
   /**
    * Execute this web service test.
    * <p>
    * This will perform the HTTP request and test the HTTP response with the {@link #response}.
    */
   @Override
   public void execute()
   {
      String uri = pool.setVariables(service);
      if (!(uri.toLowerCase().startsWith("http://") || uri.toLowerCase().startsWith("https://")))
      {
         uri = cfg.endpoint;
         if (!uri.endsWith("/"))
         {
            uri = uri + "/";
         }
         
         String srv = pool.setVariables(service);
         while (srv.startsWith("/"))
         {
            srv = srv.substring(1); 
         }
         uri = uri + srv;
      }
      
      String httpMethod = method == null ? getDefaultMethod() : method.toUpperCase();
      RequestBuilder requestBuilder = RequestBuilder.create(httpMethod).setUri(uri);

      queryParameters.forEach((name, value) -> requestBuilder.addParameter(pool.setVariables(name), 
                                                                           pool.setVariables(value)));
      
      String defaultContentType = getDefaultContentType();
      if (defaultContentType != null && !headers.containsKey("Content-Type"))
      {
         headers.put("Content-Type", defaultContentType);
      }
      
      headers.forEach((name, value) -> requestBuilder.addHeader(pool.setVariables(name), 
                                                                pool.setVariables(value)));
      StringBuilder cookieHeader = new StringBuilder();
      cookies.forEach((name, value) -> cookieHeader.append(pool.setVariables(name))
                                                   .append("=")
                                                   .append(pool.setVariables(value))
                                                   .append(";"));
      requestBuilder.addHeader("Cookie", cookieHeader.toString());
      
      if (authorization != null)
      {
         authorization.setAuthorization(requestBuilder);
      }
      
      if (requestBody != null)
      {
         requestBody.build(requestBuilder);
      }

      HttpUriRequest httpRequest = requestBuilder.build();

      StringBuilder descr = new StringBuilder();
      descr.append("REQUEST\n");
      descr.append(httpRequest.toString()).append("\n");
      for (Header h : httpRequest.getAllHeaders())
      {
         descr.append(h.toString()).append("\n");
      }
      String body = requestBody == null ? null : requestBody.getBody();
      if (body != null)
      {
         descr.append("\n");
         descr.append(prettyPrint(requestBody.getBody()));
      }
      setDescription(descr.toString());
      
      CloseableHttpResponse httpResponse = null;
      CloseableHttpClient client = this.client.get();
      try
      {
         super.setStartTime(System.currentTimeMillis());
         httpResponse = client.execute(httpRequest);
      }
      catch (Throwable t)
      {
         try
         {
            client.close();
         }
         catch (IOException e)
         {
            // ignore
         }

         t.printStackTrace();

         String msg = t.getMessage();
         while (msg == null && t != null)
         {
            t = t.getCause();
            msg = t.getMessage();
         }
         setStatus(Status.FAILED, "Request failed: " + t.getMessage());
         return;
      }
      finally
      {
         super.setEndTime(System.currentTimeMillis());
      }

      // assume passed, it will be set correctly by the response processing.
      setStatus(Status.PASSED, null);
      
      descr.append("\n\nRESPONSE\n");
      descr.append(httpResponse.getStatusLine().toString()).append("\n");
      for (Header h : httpResponse.getAllHeaders())
      {
         descr.append(h.getName() + ": " + h.getValue()).append("\n");
      }
      responseBody = readResponseBody(httpResponse);
      if (responseBody != null)
      {
         descr.append("\n");
         if (cfg.omitResponseBody)
         {
            descr.append("!!! RESPONSE BODY OMITTED !!!");
         }
         else
         {
            descr.append(prettyPrint(responseBody));
         }
      }
      setDescription(descr.toString());

      try
      {
         if (this.response != null)
         {
            try
            {
               if (!prepareResponse())
               {
                  return;
               }
               
               this.response.execute(httpResponse);
               
               if (response.getStatus() != Status.PASSED)
               {
                  // dump the entire response
                  StringBuilder dump = new StringBuilder();
                  dump.append(httpResponse.getStatusLine()).append("\n");
                  for (Header h : httpResponse.getAllHeaders())
                  {
                     dump.append(h).append("\n");
                  }
                  // do not set the response body in the status, as it may be huge.
                  
                  setStatus(response.getStatus(), response.getReason() + "\n\n" + dump);
               }
            }
            finally
            {
               releaseResponse();
            }
         }
      }
      finally
      {
         try
         {
            httpResponse.close();
         } 
         catch (IOException e1)
         {
            // ignore
         }
         
         try
         {
            client.close();
         }
         catch (IOException e)
         {
            // ignore
         }
         responseBody = null;
      }
   }
   
   /**
    * Check if the description and failure content needs to be wrapped.
    * 
    * @return   Always <code>true</code>.
    */
   @Override
   public boolean isWrappedContent()
   {
      return true;
   }
   
   /**
    * Get the actual response body.
    * 
    * @return   The {@link #responseBody}.
    */
   public String getResponseBody()
   {
      return responseBody;
   }

   /**
    * Get the test configuration.
    * 
    * @return   The {@link #cfg configuration}.
    */
   public WebSessionSettings getConfiguration()
   {
      return cfg;
   }
   
   /**
    * Set the web service authorization.
    * 
    * @param    auth
    *           The authorization to use.
    */
   public void setAuthorization(WebAuthorization auth)
   {
      this.authorization = auth;
   }

   /**
    * Set the web service.
    * 
    * @param    service
    *           The service to execute.
    */
   public void setService(String service)
   {
      if (service == null)
      {
         service = "";
      }
      
      this.service = service;
   }

   /**
    * Set the HTTP method.
    * 
    * @param    method
    *           The HTTP method.
    */
   public void setMethod(String method)
   {
      this.method = method;
   }
   
   /**
    * Set a request query parameter.
    * 
    * @param    name
    *           The parameter name.
    * @param    value
    *           The parameter value.
    */
   public void addQueryParameter(String name, String value)
   {
      this.queryParameters.put(name, value);
   }

   /**
    * Add a request header.
    * 
    * @param    name
    *           The header name.
    * @param    value
    *           The header value.
    */
   public void addHeader(String name, String value)
   {
      this.headers.put(name, value);
   }
   
   /**
    * Add a request cookie.
    * 
    * @param    name
    *           The cookie name.
    * @param    value
    *           The cookie value.
    */
   public void addCookie(String name, String value)
   {
      this.cookies.put(name, value);
   }

   /**
    * Set the HTTP request body {@link HttpRequestBody configuration}.
    *  
    * @param    body
    *           The body configuration.
    */
   public void setRequestBody(HttpRequestBody body)
   {
      this.requestBody = body;
   }

   /**
    * Set the tests to apply on the HTTP response.
    * 
    * @param    response
    *           The response test configuration.
    */
   public void setResponse(HttpResponseTest response)
   {
      this.response = response;
      response.setParent(this);
   }

   /**
    * Get the {@link #pool resource pool}.
    * 
    * @return   The {@link #pool resource pool}.
    */
   public ResourcePool getResourcePool()
   {
      return pool;
   }

   /**
    * Get the default HTTP method.
    * 
    * @return   <code>GET</code> by default.
    */
   protected String getDefaultMethod()
   {
      return "GET";
   }
   
   /**
    * Read the response body, as received after the web service request was invoked.
    * 
    * @param    response
    *           The HTTP response.
    *           
    * @return   The response body.
    */
   private String readResponseBody(HttpResponse response)
   {
      // WARNING: multipart, bytes (or any kind of response which requires additional processing) are not supported!
      try
      {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         response.getEntity().writeTo(baos);
         return new String(baos.toByteArray());
      }
      catch (IOException exc)
      {
         setStatus(Status.FAILED, "Error reading response body: " + exc.getMessage());
         exc.printStackTrace();
         return null;
      }
   }

   /**
    * Construct a SSL Context for HTTPS.
    * 
    * @return   The SSL context.
    * 
    * @throws    KeyManagementException
    * @throws    NoSuchAlgorithmException
    * @throws    KeyStoreException
    * @throws    CertificateException
    * @throws    IOException
    */
   private SSLContext getSSLContext() 
   throws KeyManagementException,
          NoSuchAlgorithmException, 
          KeyStoreException, 
          CertificateException, 
          IOException 
   {
      Boolean acceptSelfSigned = cfg.sslAcceptSelfSigned;
      Boolean acceptAll = cfg.sslAcceptAll;
      String ksFile = cfg.sslTrustStoreFile;
      String ksPwd = cfg.sslTrustStorePassword;
      
      TrustStrategy trustStrategy;
      if (!Boolean.FALSE.equals(acceptAll))
      {
         // System.out.println("Http client accepts all certificates");
         trustStrategy =  new TrustStrategy() 
         {
            @Override
            public boolean isTrusted(X509Certificate[] chain, String authType) 
            throws CertificateException
            {
               return true;
            }
         };
      }
      else if (!Boolean.FALSE.equals(acceptSelfSigned)) 
      {
         // System.out.println("Http client accepts self-signed certificates");
         trustStrategy = TrustSelfSignedStrategy.INSTANCE; 
         
      }
      else 
      {
         trustStrategy =  new TrustStrategy() 
         {
            @Override
            public boolean isTrusted(X509Certificate[] chain, String authType) 
            throws CertificateException
            {
               return false;
            }
         };
      }
      
      if (ksFile != null)
      {
         File ks = new File(ksFile);
         // System.out.println("Using trusted keystore: " + ks.getAbsolutePath());
         return new SSLContextBuilder().loadTrustMaterial(ks, 
                                                          ksPwd != null ? ksPwd.toCharArray() : null , 
                                                          trustStrategy)
                                       .build();
      }
      
      return new SSLContextBuilder().loadTrustMaterial(trustStrategy).build(); 
   }
   
   /**
    * Get the client connection manager.
    * 
    * @param    sslContext
    *           The SSL context.
    *           
    * @return   The client connection manager.
    */
   private HttpClientConnectionManager getHttpClientConnectionManager(SSLContext sslContext)
   {
      HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
      SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, 
                                                                                   hostnameVerifier);
      
      Registry<ConnectionSocketFactory> socketFactoryRegistry =
               RegistryBuilder.<ConnectionSocketFactory>create()
                              .register("http", PlainConnectionSocketFactory.getSocketFactory())
                              .register("https", sslSocketFactory)
                              .build();
      
      return new PoolingHttpClientConnectionManager(socketFactoryRegistry);
   }
   
   /**
    * Create a new HTTP client to execute the request.
    * 
    * @param    proxyHost
    *           The proxy host.
    * @param    proxyPort
    *           The proxy port.
    *           
    * @return   The HTTP client to execute the web service.
    */
   private CloseableHttpClient createHttpClient(String proxyHost, int proxyPort)
   {
      try
      {
         SSLContext sslContext = getSSLContext();
         HttpClientConnectionManager connectionManager = getHttpClientConnectionManager(sslContext);
   
         RedirectStrategy strategy = new RedirectStrategy() 
         {
            @Override
            public boolean isRedirected(HttpRequest arg0, HttpResponse arg1, HttpContext arg2)
            throws ProtocolException
            {
               return false;
            }
            
            @Override
            public HttpUriRequest getRedirect(HttpRequest arg0, HttpResponse arg1, HttpContext arg2)
            throws ProtocolException
            {
               return null;
            }
         };
         
         // 'easy' cookie mode is used so invalid cookies are not dropped
         RequestConfig defReqConfig = RequestConfig.custom().setCookieSpec("easy").build();
         HttpClientBuilder builder = HttpClientBuilder.create()
                                                      .setConnectionManager(connectionManager)
                                                      .setRedirectStrategy(strategy)
                                                      .setSSLContext(sslContext)
                                                      .setDefaultRequestConfig(defReqConfig);
         
         if (proxyHost != null)
         {
            HttpHost proxy = new HttpHost(proxyHost, proxyPort);
            builder.setProxy(proxy);
         }
         
         return builder.build();
      }
      catch (Exception ex)
      {
         // the HTTP client could not be build, fatal failure
         throw new RuntimeException(ex);
      }
   }
}
