# MethodTraceAspect.java

    /*
    ** Module   : MethodTraceAspect.java
    ** Abstract : Aspect to capture method trace data
    **
    ** Copyright (c) 2019-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------Description----------------------------------
    ** 001 ECF 20190504 Created initial version.
    ** 002 AL2 20221025 Added possibility to export to HTML UI.
    ** 003 AL2 20221027 Added summary CSV of the method tracing.
    ** 004 AL2 20221027 Fixed negative ownTime. Fixed % grand total.
    ** 005 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    */
    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.aspects.ltw;

    import java.io.*;
    import java.lang.reflect.*;
    import java.math.RoundingMode;
    import java.text.DecimalFormat;
    import java.util.*;
    import java.util.concurrent.*;
    import java.util.concurrent.atomic.*;
    import java.util.logging.*;
    import java.util.stream.Collectors;

    import org.aspectj.lang.*;
    import org.aspectj.lang.annotation.*;
    import org.aspectj.lang.annotation.Pointcut;
    import org.aspectj.lang.reflect.*;
    import com.goldencode.p2j.util.logging.*;

    /**
     * An abstract aspect to capture profile data (invocation count and aggregate time) per method.
     * The aspect is designed for load-time weaving using a concrete aspect subclass defined in an
     * {@code aop.xml} file. That concrete aspect must define a pointcut for the abstract {@link
     * #scope()} method.
     * <p>
     * Although the tracing advice implementation is designed to introduce as little overhead as
     * possible, the overhead is quite noticeable while data collection is active (less so while it
     * is disabled). This overhead will be reflected in the collected data, and it should be taken
     * into account when analyzing the results. Part of the overhead is due to the load-time weaving
     * of this aspect with the classes it is measuring. This weaving is done once, when the target
     * class is first loaded into the JVM. To minimize the impact of this one-time weave on the
     * trace results, it is recommended to perform the work flow to be measured once before starting
     * data collection, then perform it again with data collection turned on. However, this is not
     * always possible, if what is being measured is the first-time execution of a code path, where
     * future executions will be short-circuited by caching of various resources by the runtime.
     * <p>
     * Due to the aforementioned overhead which this feature introduces to running code, it is
     * recommended not to enable this feature in production deployments.
     * <p>
     * Tracing is controlled by the {@link #enableDataCollection(boolean)} method. When an active
     * tracing session is disabled, all data which has been collected up to that point is
     * post-processed and written to a CSV file, then discarded.
     */
    @Aspect
    public abstract class MethodTraceAspect
    {
       /** Logger */
       private static final CentralLogger LOG = CentralLogger.get(MethodTraceAspect.class);

       static
       {
          if (LOG.isLoggable(Level.WARNING))
          {
             LOG.warning("Method profiling is enabled; " +
                         "this feature is not intended for production use!");
          }
       }

       /** The name of the folder in which the HTML UI artifacts will be generated */
       private static String TRACE_OUTPUT = "trace";

       /** The name of the folder where the trace data will be dumped to be used by the HTML UI */
       private static String DATA_OUTPUT = "data";

       /** Fixed name of CSV output file (written to current directory) */
       private static String CSV_FILE = "trace.csv";

       /** Fixed name of summary CSV output file (written to current directory) */
       private static String SUMMARY_FILE = "trace_summary.csv";

       /** Fixed name of all methods list CSV output file (written to current directory) */
       private static String METHOD_LIST_FILE = "trace_method_list.csv";

       /** Flag which controls data collection */
       private static volatile boolean enabled = false;

       /** Start time of a trace session, in milliseconds */
       private static long traceStart = 0L;

       /** Map of profile data by method signature */
       private static final Map<Call, TraceData> traceData = new ConcurrentHashMap<>();

       /** Thread-local stack of trace data since entering the first join point in the traced code */
       private static final ThreadLocal<Deque<TraceData>> callStack = new ThreadLocal<Deque<TraceData>>()
       {
          @Override
          public Deque<TraceData> initialValue()
          {
             return new ArrayDeque<>();
          }
       };

       /** Package "buckets" for data post-processing */
       private static final String[] buckets =
       {
          // TODO: hard-coded and duplicates information in aop.xml, but how to easily extract that?
          // When adding subpackages, write them below the existent parent package (if it exists).
          // This will make it easier to match the correct bucket
          "com.goldencode.p2j.persist.",
          "com.goldencode.p2j.persist.orm.",
          "com.mchange.",
          "org.postgresql.",
          "org.h2.",
       };

       /**
        * Enable or disable data collection. Upon disabling data collection, any collected results
        * are processed and exported.
        *
        * @param   enable
        *          {@code true} to start data collection; {@code false} to stop it and process the
        *          results.
        */
       public synchronized static void enableDataCollection(boolean enable)
       {
          if (enable)
          {
             if (!enabled)
             {
                traceStart = System.currentTimeMillis();
                enabled = true;
             }

             return;
          }

          // stop collecting and process collected results

          try
          {
             // we potentially give up some accuracy in the results if a thread is finishing up its
             // collection work while we process the results, but this is done willingly to avoid
             // the overhead of synchronization while collecting trace data
             enabled = false;
             processData();
             exportHTML();
          }
          finally
          {
             // reset results
             traceData.clear();
             traceStart = 0L;
          }
       }

       /**
        * Routine used for generating a HTML UI used for better interpreting of the data. The end-user
        * interface groups the method tracing on roots and buckets.
        */
       private static void exportHTML()
       {
          DecimalFormat df = new DecimalFormat("0.00000");
          df.setRoundingMode(RoundingMode.DOWN);
          long grandTotalElapsed = 0L;

          // organize trace data into buckets, indexed by entry point (first traced) method
          Map<MethodIdentifier, BucketData> bucketsByRoot = getBucketsByRoot();
          for (BucketData bucketData : bucketsByRoot.values())
          {
             grandTotalElapsed += bucketData.getElapsed();
          }

          /* Ensure the intermediary folders exist */
          new File(TRACE_OUTPUT + File.separator + DATA_OUTPUT).mkdirs();

          /* A meta file is used to track: the names of the buckets, the roots and the total elapsed time */
          String metaFile = TRACE_OUTPUT + File.separator + DATA_OUTPUT + File.separator + "meta.js";
          try (BufferedWriter metaWriter = new BufferedWriter(new FileWriter(metaFile)))
          {
             // total elapsed time
             metaWriter.append("var ").append("elapsed = " + grandTotalElapsed).append(";");
             metaWriter.newLine();
             // name of the buckets
             String bucketsArray = Arrays.asList(buckets).stream()
                      .map((value) -> "\"" + value + "\"").collect(Collectors.joining(", "));
             metaWriter.append("var ").append("buckets = ").append("[" + bucketsArray + "];");
             metaWriter.newLine();

             List<String> roots = new ArrayList<>();
             List<String> scriptInject = new ArrayList<>();

             for (Map.Entry<MethodIdentifier, BucketData> entry : bucketsByRoot.entrySet())
             {
                String root = entry.getKey().getShortName();
                BucketData data = entry.getValue();
                if (data.getElapsed() < 1000000)
                {
                   // avoid exporting scenarios executed in < 1 ms
                   continue;
                }

                // each root will have tracing information in a folder inside the data folder
                String rootFile = TRACE_OUTPUT + File.separator + DATA_OUTPUT + File.separator + root;
                new File(rootFile).mkdirs();
                roots.add(root);

                for (String bucket : buckets)
                {
                   // keep names simple, without additional .
                   // the decision to export each tracing information in separate JS is to ensure that
                   // we can load the data statically (no need of file server to use the interface)
                   String jsName = bucket.replace(".", "") + ".js";
                   String bucketFile = rootFile + File.separator + jsName;
                   try (BufferedWriter bucketWriter = new BufferedWriter(new FileWriter(bucketFile)))
                   {
                      // traceData map is initialized in the loader; here we just add data for each bucket
                      bucketWriter.append("traceData[").append("\"" + root + "\"")
                                  .append("][").append("\"" + bucket + "\"").append("] = {");
                      long bucketElapsed = data.getTraces(bucket).getElapsed();

                      // meta data like own_time, proc_time and grand_time
                      bucketWriter.append("\"own_time\" : ")
                                  .append("" + bucketElapsed).append(", ");
                      bucketWriter.append("\"proc_time\" : ")
                                  .append(df.format(bucketElapsed / (double) data.getElapsed())).append(", ");
                      bucketWriter.append("\"grand_time\" : ")
                                  .append(df.format(bucketElapsed / (double) grandTotalElapsed)).append(", ");

                      // method tracing is an array of objects
                      bucketWriter.append("\"traces\" : [");
                      bucketWriter.newLine();
                      for (TraceData trace : data.getTraces(bucket).getTraces())
                      {
                         bucketWriter.append("{");
                         MethodIdentifier callee = trace.callee;
                         bucketWriter.append("\"name\" : ");
                         if (callee != null)
                         {
                            Method method = trace.callee.method;
                            bucketWriter.append('"').append(method.getDeclaringClass().getName())
                                        .append('.').append(method.getName()).append('"');
                         }
                         else
                         {
                            bucketWriter.write("\"All others...\"");
                         }
                         bucketWriter.append(',').append("\"count\" : ")
                                     .append(Long.toString(trace.count.get()))
                                     .append(',').append("\"elapsed\" : ")
                                     .append(Long.toString(trace.elapsed.get()))
                                     .append(',').append("\"time_proc\" : ")
                                     .append(Double.toString(trace.elapsed.get() / data.getElapsed()))
                                     .append(',').append("\"owntime\" : ")
                                     .append(Long.toString(trace.getOwnTime()))
                                     .append(',').append("\"owntime_proc\" : ")
                                     .append(Double.toString(trace.getOwnTime() / data.getElapsed()))
                                     .append(',').append("\"grand_proc\" : ")
                                     .append(Double.toString(trace.getOwnTime() / (double) grandTotalElapsed));
                         bucketWriter.append("},");
                         bucketWriter.newLine();
                      }

                      bucketWriter.append("]};");
                   }

                   // make sure we keep track of which js files we generates; these should be statically imported
                   scriptInject.add(DATA_OUTPUT + File.separator + root + File.separator + jsName);
                }
             }
             // generate required loader (JS), run-time (JS) and bootstrap (CSS)
             generateAux(roots);
             String rootsArray = roots.stream()
                      .map((value) -> "\"" + value + "\"").collect(Collectors.joining(", "));
             metaWriter.append("var ").append("roots = ").append("[" + rootsArray + "];");

             // statically import each script directly from the HTML
             // if we use JS module import or fetch, we need a file server to satisfy CORS
             // we have control over HTML creation: just bump all JS there
             InputStream is = MethodTraceAspect.class.getResourceAsStream("web/method_tracing.html");
             String inputHtml = new BufferedReader(new InputStreamReader(is)).lines().collect(Collectors.joining("\n"));
             String extraScripts = scriptInject.stream()
                      .map((value) -> "<script type=\"text/javascript\" src=\"" + value + "\" defer></script>")
                      .collect(Collectors.joining("\n\t\t"));
             inputHtml = inputHtml.replace("<!-- Extra scripts -->", extraScripts);

             String outputHtmlPath = TRACE_OUTPUT + File.separator + "index.html";
             try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(outputHtmlPath)))
             {
                outputStream.write(inputHtml);
             }
          }
          catch (IOException e)
          {
             LOG.severe("", e);
          }
       }

       /**
        * Helper method for HTML UI generation. This ensures that the requires JS and CSS are generated
        * for the index HTML. Otherwise, functionality may lack.
        *
        * @param   roots
        *          A list of all JS files we created containing information about multiple traced methods.
        *          These should be dumped in the loader JS which ensures that the data is loaded in the
        *          JS memory.
        */
       private static void generateAux(List<String> roots)
       throws IOException
       {
          new File(TRACE_OUTPUT + File.separator + "js").mkdirs();
          new File(TRACE_OUTPUT + File.separator + "css").mkdirs();
          String loaderFile = TRACE_OUTPUT + File.separator + "js" + File.separator + "loader.js";
          try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(loaderFile)))
          {
             outputStream.write("var traceData = {};");
             outputStream.newLine();
             for (String root : roots)
             {
                outputStream.write("traceData[\"" + root + "\"] = {};");
                outputStream.newLine();
             }
          }

          String traceFile = TRACE_OUTPUT + File.separator + "js" + File.separator + "trace.js";
          InputStream is = MethodTraceAspect.class.getResourceAsStream("web/trace.js");
          String traceJS = new BufferedReader(new InputStreamReader(is)).lines().collect(Collectors.joining("\n"));
          try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(traceFile)))
          {
             outputStream.write(traceJS);
          }

          String bootstrapFile = TRACE_OUTPUT + File.separator + "css" + File.separator + "bootstrap.min.css";
          InputStream bis = MethodTraceAspect.class.getResourceAsStream("web/bootstrap.min.css");
          String bootstrapCss = new BufferedReader(new InputStreamReader(bis)).lines().collect(Collectors.joining("\n"));
          try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(bootstrapFile)))
          {
             outputStream.write(bootstrapCss);
          }

       }

       /**
        * Utility method used by both HTML and CSV UI. This groups all tracked bucket data
        * per "root" (the method which determined the execution of all methods from the
        * bucket).
        *
        * @return   A map of bucket data grouped by the root / scenario.
        */
       private static Map<MethodIdentifier, BucketData> getBucketsByRoot()
       {
          Map<MethodIdentifier, BucketData> bucketsByRoot = new HashMap<>();

          for (Map.Entry<Call, TraceData> e : traceData.entrySet())
          {
             Call call = e.getKey();
             TraceData trace = e.getValue();

             MethodIdentifier caller = call.caller;
             MethodIdentifier callee = call.callee;

             String callerBucket = matchBucket(caller);
             String calleeBucket = matchBucket(callee);

             boolean transition = !Objects.equals(callerBucket, calleeBucket);

             MethodIdentifier root = call.root;
             BucketData bucketData = bucketsByRoot.get(root);

             if (bucketData == null)
             {
                bucketData = new BucketData();
                bucketsByRoot.put(root, bucketData);
             }

             if (calleeBucket != null)
             {
                bucketData.addTrace(calleeBucket, trace, transition, root.equals(callee));
             }
          }
          return bucketsByRoot;
       }

       /**
        * Post-process the collected trace data and write the result to a CSV file. This involves
        * organizing the data into "buckets", each representing a transition from one code framework
        * to another. Only the trace data for the first method call in each new bucket is retained,
        * as it represents the aggregate time spent in that method and all of its called methods.
        * This reduced data set is then sorted in descending order by aggregate time spent in the
        * outermost method call across each set of buckets. This sorted data is then written out to
        * a CSV file.
        */
       private static void processData()
       {
          // get overall elapsed trace time in milliseconds.
          long elapsed = System.currentTimeMillis() - traceStart;

          // organize trace data into buckets, indexed by entry point (first traced) method

          Map<MethodIdentifier, BucketData> bucketsByRoot = getBucketsByRoot();

          // compute grand totals by bucket

          int len = buckets.length;

          long grandTotalElapsed = 0L;

          long[] totals = new long[len];

          for (BucketData bucketData : bucketsByRoot.values())
          {

             long across = 0;
             for (int i = 0; i < len; i++)
             {
                String bucket = buckets[i];
                TraceBucket traces = bucketData.getTraces(bucket);
                totals[i] += traces.getElapsed();
                across += traces.getElapsed();
             }

             // this is not ideal; the bucket data elapsed type can be sometimes different than the across time
             // as it includes the time spent in packages not tracked. across is always correct as it computes
             // the time spent in the profiled packages. However, this should be solved by adding all packages
             // needed by the tracker not to rely on across, but on the total time.
             if (across != bucketData.getElapsed())
             {
                grandTotalElapsed += across;
             }
             else
             {
                grandTotalElapsed += bucketData.getElapsed();
             }
          }

          // sort bucket groups by aggregate elapsed time of longest bucket
          SortedSet<BucketData> sortedBucketData = new TreeSet<>(new ByTotalTime());
          sortedBucketData.addAll(bucketsByRoot.values());

          // write trace data out to CSV, one column group per bucket

          try (BufferedWriter bw = new BufferedWriter(new FileWriter(CSV_FILE));
               BufferedWriter summary = new BufferedWriter(new FileWriter(SUMMARY_FILE));
               BufferedWriter allMethods = new BufferedWriter(new FileWriter(METHOD_LIST_FILE));)
          {
             // elapsed trace time
             bw.append("\"Elapsed trace time (ms):\",").append(Long.toString(elapsed)).append(",");
             summary.append("\"Elapsed trace time (ms):\",").append(Long.toString(elapsed)).append(",");
             bw.newLine();
             bw.newLine();
             summary.newLine();
             summary.newLine();

             summary.append('"').append("Root").append("\",");
             summary.append("\"Count\",").append("\"Agg Time (nano)\",").append("\"Own Time (nano)\",");

             // column headings
             for (String bucket : buckets)
             {
                String shortBucket = bucket.substring(0, bucket.length() - 1);
                bw.append('"').append(shortBucket).append("\",");
                bw.append("\"Count\",").append("\"Agg Time (nano)\",").append("\"% of Total\",");
                bw.append("\"Own Time (nano)\",").append("\"% of Total\",").append("\"% of Grand Total\",");

                summary.append('"').append(shortBucket + " (own time)").append("\",\"");
                summary.append(shortBucket + " (% of Total)").append("\",");
             }
             bw.newLine();
             summary.newLine();

             for (BucketData bucketData : sortedBucketData)
             {
                double totalElapsed = bucketData.getElapsed();

                for (TraceData[] row : bucketData.asRows())
                {
                   for (int i = 0; i < len; i++)
                   {
                      TraceData trace = row[i];

                      if (trace == null)
                      {
                         bw.write("\"\",,,,,,,");

                         continue;
                      }

                      MethodIdentifier callee = trace.callee;
                      if (callee != null)
                      {
                         Method method = trace.callee.method;
                         bw.write('"');
                         bw.write(method.getDeclaringClass().getName());
                         bw.write('.');
                         bw.write(method.getName());
                         bw.write('"');
                      }
                      else
                      {
                         bw.write("\"All others...\"");
                      }
                      bw.write(',');
                      bw.write(Long.toString(trace.count.get()));
                      bw.write(',');
                      bw.write(Long.toString(trace.elapsed.get()));
                      bw.write(',');
                      bw.write(Double.toString(trace.elapsed.get() / totalElapsed));
                      bw.write(',');
                      // use getOwnTime; using directly the variable ownTime may result in
                      // negative results, as the method tracing can close without finishing
                      // the own time computing
                      bw.write(Long.toString(trace.getOwnTime()));
                      bw.write(',');
                      bw.write(Double.toString(trace.getOwnTime() / totalElapsed));
                      bw.write(',');
                      bw.write(Double.toString(trace.getOwnTime() / (double) grandTotalElapsed));
                      bw.write(',');
                   }

                   bw.newLine();
                }

                // write statistics only about the root in the summary
                TraceData rootTrace = bucketData.root;
                if (rootTrace != null)
                {
                   if (rootTrace.callee != null)
                   {
                      Method rootMethod = rootTrace.callee.method;
                      summary.append('"').append(rootMethod.getDeclaringClass().getName());
                      summary.append('.').append(rootMethod.getName()).append("\",");
                   }
                   else
                   {
                      bw.write("\"Other...\"");
                   }
                   summary.append(Long.toString(rootTrace.count.get())).append(",");
                   summary.append(Long.toString(rootTrace.elapsed.get())).append(",");
                   summary.append(Long.toString(rootTrace.getOwnTime())).append(",");
                }
                else
                {
                   summary.append(",,,,");
                }

                // write totals
                Iterator<TraceBucket> iter = bucketData.traces();
                while (iter.hasNext())
                {
                   TraceBucket bucket = iter.next();

                   long bucketElapsed = bucket.getElapsed();

                   bw.write("\"   Bucket Totals...\",,,,");
                   summary.append(Long.toString(bucketElapsed)).append(",");
                   summary.append(Double.toString(bucketElapsed / totalElapsed)).append(",");
                   bw.write(Long.toString(bucketElapsed));
                   bw.write(',');
                   bw.write(Double.toString(bucketElapsed / totalElapsed));
                   bw.write(',');
                   bw.write(Double.toString(bucketElapsed / (double) grandTotalElapsed));
                   bw.write(',');
                }

                bw.newLine();
                bw.newLine();
                summary.newLine();
             }

             writeAllMethods(allMethods);

             // write grand totals
             writeGrandTotals(bw, len, totals, grandTotalElapsed);
             writeGrandTotals(summary, len, totals, grandTotalElapsed);
          }
          catch (IOException exc)
          {
             LOG.log(Level.SEVERE, "I/O error writing trace data CSV", exc);
          }
       }

       /**
        * Utility method for dumping all the methods traces together with count and own time.
        *
        * @param   bw
        *          The writter where this statistic should be written.
        */
       private static void writeAllMethods(BufferedWriter bw)
       throws IOException
       {
          Map<MethodIdentifier, TraceData> dataPerMethod = aggregatePerMethod();

          bw.append("\"Package\",\"Class\",\"Method name\",\"Count\",\"Own Time\"");
          bw.newLine();

          List<String> lines = new ArrayList<>();
          for (Map.Entry<MethodIdentifier, TraceData> entry : dataPerMethod.entrySet())
          {
             StringBuilder sb = new StringBuilder();
             String className = entry.getKey().method.getDeclaringClass().getName();
             sb.append('"').append(className.substring(0, className.lastIndexOf(".")));
             sb.append("\",");
             sb.append('"').append(className).append("\",");
             sb.append('"').append(className);
             sb.append('.').append(entry.getKey().method.getName());
             Class<?>[] params = entry.getKey().method.getParameterTypes();
             sb.append("(");
             for (int i = 0; i < params.length; i++)
             {
                if (i != 0)
                {
                   sb.append(",");
                }
                sb.append(params[i].getSimpleName());
             }
             sb.append(")").append("\",");

             sb.append(Long.toString(entry.getValue().count.get())).append(",");
             sb.append(Long.toString(entry.getValue().getOwnTime())).append(",");
             lines.add(sb.toString());
          }

          Collections.sort(lines);

          for (String line : lines)
          {
             bw.append(line);
             bw.newLine();
          }

          bw.newLine();
       }

       /**
        * Retrieve a map between each tracked method and its trace data. This is an aggregate
        * of all buckets. A method can appear in multiple buckets - here we compute the
        * total time. Don't use the caller of the trace data as there is none - a trace
        * data holds information about multiple traces. Its own time and count is in fact
        * a total of all traces with the same method identifier.
        *
        * @return   A map between all methods and their total own time and count across
        *           all buckets.
        */
       private static Map<MethodIdentifier, TraceData> aggregatePerMethod()
       {
          Map<MethodIdentifier, TraceData> dataByMethod = new HashMap<>();

          for (Map.Entry<Call, TraceData> e : traceData.entrySet())
          {
             Call call = e.getKey();
             TraceData trace = e.getValue();

             MethodIdentifier callee = call.callee;
             TraceData methodData = dataByMethod.get(callee);

             if (methodData == null)
             {
                methodData = new TraceData(null, callee);
                dataByMethod.put(callee, methodData);
             }
             methodData.count.addAndGet(trace.count.get());
             // use getOwnTime to avoid negative results; this can happen when the tracing stops
             // before finishing the computation of own time
             methodData.ownTime.addAndGet(trace.getOwnTime());
          }
          return dataByMethod;
       }

       /**
        * Utility method for CSV exporting of the data. More exactly, this prints the grand totals
        * pe bucket fort he active method tracing.
        *
        * @param   bw
        *          The writer where the grand totals shoudl be written
        * @param   len
        *          The number of buckets
        * @param   totals
        *          The total values
        * @param   grandTotalElapsed
        *          The grand total elapsed
        */
       private static void writeGrandTotals(BufferedWriter bw, int len, long[] totals, long grandTotalElapsed)
       throws IOException
       {
          bw.newLine();

          bw.write("\"Grand Totals\",");
          bw.newLine();
          bw.write("\"Bucket\",,\"Agg Own Time (nano)\",\"% of Grand Total\"");
          bw.newLine();

          for (int i = 0; i < len; i++)
          {
             bw.append('"').append(buckets[i]).append("\",,").append(Long.toString(totals[i]));
             bw.append(',').append(Double.toString(totals[i] / (double) grandTotalElapsed)).append(',');
             bw.newLine();
          }
       }

       /**
        * Attempt to match a method's declaring class with a data bucket identifier.
        *
        * @param   ident
        *          Method identifier.
        *
        * @return  The matching bucket identifer string, if a match was found, else {@code null}.
        */
       private static String matchBucket(MethodIdentifier ident)
       {
          Method method = ident.method;

          if (method == null)
          {
             return null;
          }

          String name = method.getDeclaringClass().getName();
          String bestMatch = null;
          for (String bucket : buckets)
          {
             if (name.startsWith(bucket))
             {
                // this depends on the fact that all packages and subpackages from buckets
                // are placed in the correct order
                bestMatch = bucket;
             }
          }

          return bestMatch;
       }

       /**
        * An abstract pointcut which should be defined for load-time weaving inside a {@code
        * concrete-aspect} element within {@code META-INF/aop.xml}.
        */
       @Pointcut
       public abstract void scope();

       /**
        * The primary tracing advice which collects trace information about calls to the target
        * methods. Here we attempt the following with minimal overhead:
        * <ul>
        * <li>Determine the caller method.
        * <li>Determine the callee method.
        * <li>Measure the elapsed number of nanoseconds it takes for the callee method to execute
        *     and add that value to a running aggregate for this method and subtract it from the
        *     own time of our calling method, if any.
        * <li>Increment the number of times this caller invoked this callee.
        * </ul>
        * <p>
        * Data is collected safely across all threads on which calls to the traced methods are made.
        * <p>
        * The caller method is determined using a thread-local stack, which is cleared any time this
        * advice is entered after data collection has been halted. Note that the relationship between
        * caller and callee may represent an actual, direct invocation if both methods are crosscut
        * by the defined pointcut. Otherwise, if intermediate method invocations are not crosscut,
        * the recorded "caller" may actually represent some "ancestor" method further up the actual
        * call stack. It is assumed that this potential "gap" between caller and callee is understood
        * by and acceptable to the definer of the pointcut.
        *
        * @param   joinPoint
        *          Join point which represents the method being traced.
        *
        * @return  Object, if any, returned by the method being traced.
        *
        * @throws  Throwable
        *          if an exception is thrown by the method being traced.
        */
       @Around("scope()")
       public Object trace(ProceedingJoinPoint joinPoint)
       throws Throwable
       {
          if (!enabled)
          {
             // clean up any work in process from earlier tracing session
             callStack.remove();

             return joinPoint.proceed();
          }

          Deque<TraceData> stack = callStack.get();
          boolean outer = stack.isEmpty();

          JoinPoint.StaticPart staticPart = joinPoint.getStaticPart();
          MethodSignature sig = (MethodSignature) staticPart.getSignature();

          TraceData parent = outer ? null : stack.peek();
          MethodIdentifier caller = outer ? MethodIdentifier.UNTRACED : parent.callee;
          MethodIdentifier callee = new MethodIdentifier(sig.getMethod(), staticPart.getId());
          MethodIdentifier root = outer ? callee : stack.peekLast().callee;

          // path from root to callee (these are the same if this is the first traced call)
          Call path = new Call(root, caller, callee);

          // look up the aggregate trace data for this combination of caller and callee, if any such
          // information already has been initialized; if not found, initialize it
          TraceData data = traceData.get(path);
          if (data == null)
          {
             data = new TraceData(caller, callee);
             TraceData existing = traceData.putIfAbsent(path, data);
             if (existing != null)
             {
                data = existing;
             }
          }

          // emulate this thread's call stack, so nested calls have access to calling context
          stack.push(data);

          Object returnObject;
          long start = System.nanoTime();

          try
          {
             returnObject = joinPoint.proceed();
          }
          finally
          {
             // update invocation count and aggregate elapsed time for this call
             long elapsed = System.nanoTime() - start;

             if (enabled)
             {
                data.count.incrementAndGet();
                data.elapsed.addAndGet(elapsed);
                data.ownTime.addAndGet(elapsed);

                if (parent != null)
                {
                   // subtract my elapsed time from my parent's own time
                   // NOTE that this happens only when the tracing is enabled. This can
                   // remain negative at the end of the tracing; use getOwnTime when enabled is false
                   parent.ownTime.addAndGet(-elapsed);
                }

                stack.pop();
             }
             else
             {
                callStack.remove();
             }
          }

          return returnObject;
       }

       /**
        * A wrapper for a method, optimized for use as a hash map key. {@code
        * java.lang.reflect.Method} is not used directly, because its {@code hashCode()}
        * implementation operates on string data, making it slow as a hash map key.
        */
       private static class MethodIdentifier
       {
          /** Singleton instance meant to represent an untraced method */
          static final MethodIdentifier UNTRACED = new MethodIdentifier();

          /** The wrapped method */
          private final Method method;

          /** Identifier assigned by AspectJ, unique within the method's class */
          private final int id;

          /** Cached hash code */
          private final int hash;

          /**
           * Constructor which computes and caches hash code.
           *
           * @param   method
           *          The wrapped method.
           * @param   id
           *          Identifier assigned by AspectJ, unique within the method's class.
           */
          MethodIdentifier(Method method, int id)
          {
             this.method = method;
             this.id = id;

             // we calculate hash using the method's class because Method's hashCode() implementation
             // seems pretty inefficient; Class just inherits Object.hashCode()
             // the jointpoint's static part id already gives us a unique identifier for a method
             // within a type
             Class<?> type = method.getDeclaringClass();
             int result = 17;
             result = 37 * result + id;
             this.hash = 37 * result + type.hashCode();
          }

          /**
           * Private default constructor intended only to create UNKNOWN singleton.
           */
          private MethodIdentifier()
          {
             this.method = null;
             this.hash = 0;
             this.id = -1;
          }

          /**
           * Fast implementation which returns the cached hash code.
           *
           * @return  Hash code.
           */
          @Override
          public final int hashCode()
          {
             return hash;
          }

          /**
           * Implementation which is consistent with overridden {@code hashCode()} method.
           *
           * @param   o
           *          Object with which to compare this instance.
           *
           * @return  {@code true} if the given instance is equivalent to this instance.
           */
          @Override
          public final boolean equals(Object o)
          {
             MethodIdentifier that = (MethodIdentifier) o;

             // quick out
             if (this.id != that.id)
             {
                return false;
             }

             if (this == UNTRACED || that == UNTRACED)
             {
                // should only be one instance of UNKNOWN
                return this == that;
             }

             // we use Class.equals() to be consistent with our implementation of hashCode
             Class<?> thisType = this.method.getDeclaringClass();
             Class<?> thatType = that.method.getDeclaringClass();

             // short-circuit, direct comparison is safe, since Class does not override
             // Object.equals(), which just uses reference identity
             return thisType == thatType;
          }

          /**
           * Compose a string representation of this object, primarily for debugging.
           *
           * @return  String representation of this object.
           */
          @Override
          public String toString()
          {
             return (method == null ? "UNKNOWN" : method.toString()) + " <" + id + ">";
          }

          /**
           * Retrieve a simple, yet not that descriptive name. This includes only
           * the class name, without packages.
           *
           * @return   A shorter representative name.
           */
          public String getShortName()
          {
             return method.getDeclaringClass().getSimpleName().replace(".", "").replace("$", "") +
                    "_" + method.getName() + "_" + id;
          }
       }

       /**
        * Representation of a method invocation which identifies the caller and the callee methods,
        * optimized for use as a hash map key.
        */
       private static class Call
       {
          /** First method in call path traced */
          private final MethodIdentifier root;

          /** Caller method */
          private final MethodIdentifier caller;

          /** Callee method */
          private final MethodIdentifier callee;

          /** Cached hash */
          private final int hash;

          /**
           * Constructor which computes and caches hash code.
           *
           * @param   root
           *          First method in call path traced.
           * @param   caller
           *          Caller method.
           * @param   callee
           *          Callee method.
           */
          Call(MethodIdentifier root, MethodIdentifier caller, MethodIdentifier callee)
          {
             this.root = root;
             this.caller = caller;
             this.callee = callee;

             int result = 17;
             result = 37 * result + root.hashCode();
             result = 37 * result + caller.hashCode();
             this.hash = 37 * result + callee.hashCode();
          }

          /**
           * Fast implementation which returns the cached hash code.
           *
           * @return  Hash code.
           */
          @Override
          public int hashCode()
          {
             return hash;
          }

          /**
           * Implementation which is consistent with overridden {@code hashCode()} method.
           *
           * @param   o
           *          Object with which to compare this instance.
           *
           * @return  {@code true} if the given instance is equivalent to this instance.
           */
          @Override
          public boolean equals(Object o)
          {
             Call that = (Call) o;

             return this.root.equals(that.root) &&
                    this.caller.equals(that.caller) &&
                    this.callee.equals(that.callee);
          }

          /**
           * Compose a string representation of this object, primarily for debugging.
           *
           * @return  String representation of this object.
           */
          @Override
          public String toString()
          {
             return root.toString() + "..." + System.lineSeparator() +
                    "   " + caller + " ---> " + callee;
          }
       }

       /**
        * An interface with one method to return elapsed time.
        */
       private static interface ElapsedTimeable
       {
          /**
           * Get elapsed time.
           *
           * @return  Elapsed time.
           */
          public long getElapsed();
       }

       /**
        * Data structure containing invocation count and aggregate elapsed time for a method call.
        * A method call implies a specific combination of caller method and callee method, but only
        * the callee method is stored within this class.
        * <p>
        * The numeric fields which track invocation count and elapsed time are intended to be
        * accessed directly by outside code, potentially on different threads. Updates to these
        * fields individually are guaranteed to be atomic, but updates to all fields are not. Thus,
        * there is a small window of opportunity for some of these fields and not others to be
        * updated as data collection ends and the data is post-processed. This may introduce a small
        * inconsistency in the data. However, given the purpose of profiling to get an overview of
        * many method invocations, this potential for minor inconsistency is deemed acceptable, in
        * order to avoid the overhead of thread synchronization during data collection.
        */
       private static class TraceData
       implements ElapsedTimeable
       {
          /** Number of times method has been invoked */
          final AtomicLong count = new AtomicLong(0);

          /** Aggregate elapsed time spent in method (includes aspect overhead and called methods) */
          final AtomicLong elapsed = new AtomicLong(0);

          /** Aggregate own time spent in method, not including called (traced) methods */
          final AtomicLong ownTime = new AtomicLong(0);

          /** The invoking method */
          private final MethodIdentifier caller;

          /** The invoked method */
          private final MethodIdentifier callee;

          /**
           * Constructor which accepts an identifier of the invoked method.
           *
           * @param   caller
           *          Invoking method.
           * @param   callee
           *          Invoked method.
           */
          TraceData(MethodIdentifier caller, MethodIdentifier callee)
          {
             this.caller = caller;
             this.callee = callee;
          }

          /**
           * Get the aggregate elapsed time for all method calls held by this trace data.
           *
           * @return  Aggregate elapsed time.
           */
          @Override
          public long getElapsed()
          {
             return elapsed.get();
          }

          /**
           * Retrieve the own time in a safe manner. Note that own time can be negative
           * as tracing can be closed before adding the total time to a method. The sub-calls
           * are subtracting their agg time from parent's own time, so methods which do not
           * end (due to tracing close) are having a negative own time. Which an error margin,
           * we can presume that the own time is 0, as the agg time is spent only in
           * sub-calls.
           *
           * @return   The own time of the trace data or {@code 0} if the own time is negative.
           */
          public long getOwnTime()
          {
             long snap = ownTime.get();
             return snap < 0 ? 0 : snap;
          }

          /**
           * Debug string representation.
           *
           * @return  Debug information.
           */
          public String toString()
          {
             double msElapsed = elapsed.get() / 1000000.0;
             double msOwnTime = getOwnTime() / 1000000.0;
             long c = count.get();
             double avgElapsed = msElapsed / c;
             double avgOwnTime = msOwnTime / c;

             return "invoked: " + c +
                    "; elapsed: " + msElapsed +
                    "ms; own time: " + msOwnTime +
                    " ms; avg elapsed: " + avgElapsed +
                    " ms; avg own time: " + avgOwnTime + "ms" +
                    callee != null ? ("; " + caller.method + " ---> " + callee.method) : "";
          }
       }

       /**
        * A comparator which sorts trace data objects into descending order by total elapsed time.
        */
       private static class ByTotalTime
       implements Comparator<ElapsedTimeable>
       {
          @Override
          public int compare(ElapsedTimeable o1, ElapsedTimeable o2)
          {
             long diff = o1.getElapsed() - o2.getElapsed();

             return diff == 0L ? 0 : diff < 0L ? 1 : -1;
          }
       }

       /**
        * A class which organizes method trace data into specific buckets.
        */
       private static class BucketData
       implements ElapsedTimeable
       {
          /** Map of bucket names to buckets of trace data */
          private final Map<String, TraceBucket> bucketData = new LinkedHashMap<>();

          /** Elapsed time of the root method call, including all its nested calls */
          private long elapsed = 0L;

          /** The root which determined this bucket */
          private TraceData root = null;

          /**
           * Constructor.
           */
          BucketData()
          {
             // prime the map with empty trace data sets
             for (String bucket : buckets)
             {
                bucketData.put(bucket, new TraceBucket());
             }
          }

          /**
           * Add aggregated trace data for all calls to a particular method to the given bucket.
           *
           * @param   bucket
           *          Name of bucket.
           * @param   trace
           *          Aggregated trace data for a method.
           * @param   transition
           *          {@code true} if the method call captured by this trace represents a call into
           *          this bucket from another or from an untraced caller, else {@code false}.
           * @param   isRoot
           *          {@code true} if this trace represents the root (first traced) method, else
           *          {@code false}. A root method may call zero or more other traced methods.
           *          There can be only one (and there must be one) root trace per bucket data
           *          instance. Any other traces represent nested method calls in a graph originating
           *          from the root method.
           */
          void addTrace(String bucket, TraceData trace, boolean transition, boolean isRoot)
          {
             TraceBucket traces = bucketData.get(bucket);

             traces.addTrace(trace, transition);

             if (isRoot)
             {
                this.root = trace;
                elapsed = trace.elapsed.get();
             }
          }

          /**
           * Get the elapsed time of the longest bucket.
           *
           * @return  Elapsed time of the longest bucket, in nanoseconds.
           */
          @Override
          public long getElapsed()
          {
             return elapsed;
          }

          /**
           * Get the trace data for the given bucket.
           *
           * @param   bucket
           *          Name of bucket.
           *
           * @return  Trace data associated with the given bucket.
           */
          TraceBucket getTraces(String bucket)
          {
             return bucketData.get(bucket);
          }

          /**
           * Get an iterator on the trace buckets.
           *
           * @return  Trace bucket iterator.
           */
          Iterator<TraceBucket> traces()
          {
             return bucketData.values().iterator();
          }

          /**
           * Return all the stored traces across buckets as rows. Each row is an array of {@code
           * TraceData} objects. The number of elements in the array matches the number of buckets
           * into which the data is being organized. Since the buckets may contain a varying number
           * of traces, {@code null} is inserted into the array in place of a missing trace.
           * <p>
           * There is no implied, direct caller to callee relationship between the methods across the
           * columns of any particular row.
           *
           * @return  A list of {@code TraceData} arrays. The number of rows matches the number of
           *          traces in the largest bucket.
           */
          List<TraceData[]> asRows()
          {
             List<Iterator<TraceData>> iters = new ArrayList<>();
             for (TraceBucket traces : bucketData.values())
             {
                iters.add(traces.getTraces().iterator());
             }

             int len = bucketData.size();

             List<TraceData[]> rows = new ArrayList<>();

             while (true)
             {
                boolean anyHasNext = false;

                for (Iterator<TraceData> iter : iters)
                {
                   if (iter.hasNext())
                   {
                      // the buckets may have a varying number of traces; keep going while any
                      // bucket has more traces
                      anyHasNext = true;
                   }
                }

                if (!anyHasNext)
                {
                   break;
                }

                TraceData[] row = new TraceData[len];
                rows.add(row);

                for (int i = 0; i < len; i++)
                {
                   Iterator<TraceData> iter = iters.get(i);
                   row[i] = iter.hasNext() ? iter.next() : null;
                }
             }

             return rows;
          }
       }

       /**
        * Trace data for a single analysis bucket, including the trace data itself and an aggregate
        * of all the traces' own times.
        */
       private static class TraceBucket
       {
          /** Trace data sorted in descendig order by full elapsed time of each call */
          private final SortedSet<TraceData> traces = new TreeSet<>(new ByTotalTime());

          /** Elapsed time spent net of called (traced) methods (own time) */
          private long elapsed = 0L;

          /** Trace data object representing all non-transition trace data */
          private TraceData allOthers = null;

          /**
           * Add a trace and add its own time to the aggregate own time for this analysis bucket.
           *
           * @param   trace
           *          Trace data for a method call.
           * @param   transition
           *          {@code true} if the method call captured by this trace represents a call into
           *          this bucket from another or from an untraced caller, else {@code false}.
           */
          void addTrace(TraceData trace, boolean transition)
          {
             if (transition)
             {
                traces.add(trace);
             }
             else
             {
                if (allOthers == null)
                {
                   allOthers = new TraceData(null, null);
                   traces.add(allOthers);
                }

                // use getOwnTime; if trace was not closed by the tracing, the
                // real ownTime may be negative. This is not of interest here
                allOthers.ownTime.addAndGet(trace.getOwnTime());
             }

             elapsed += trace.getOwnTime();
          }

          /**
           * Get the set of traces in this analysis bucket, sorted in descending order by full,
           * elapsed times.
           *
           * @return  All trace data in this bucket.
           */
          SortedSet<TraceData> getTraces()
          {
             return traces;
          }

          /**
           * Get the aggregate own times of all traces in this analysis bucket.
           *
           * @return  Aggregate of all traces' own times.
           */
          long getElapsed()
          {
             return elapsed;
          }
       }
    }