MethodTraceAspect.java

/*
** Module   : MethodTraceAspect.java
** Abstract : Aspect to capture method trace data
**
** Copyright (c) 2019-2023, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------Description----------------------------------
** 001 ECF 20190504 Created initial version.
** 002 AL2 20221025 Added possibility to export to HTML UI.
** 003 AL2 20221027 Added summary CSV of the method tracing.
** 004 AL2 20221027 Fixed negative ownTime. Fixed % grand total.
** 005 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
**
**   0. Attribution Requirement.
**
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
**
**   1. No License To Use Trademarks.
**
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
**
**   2. No Misrepresentation of Affiliation.
**
**     You may not represent yourself as Golden Code Development Corporation or FWD.
**
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
**
**   3. No Misrepresentation of Source or Origin.
**
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.aspects.ltw;

import java.io.*;
import java.lang.reflect.*;
import java.math.RoundingMode;
import java.text.DecimalFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.logging.*;
import java.util.stream.Collectors;

import org.aspectj.lang.*;
import org.aspectj.lang.annotation.*;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.*;
import com.goldencode.p2j.util.logging.*;

/**
 * An abstract aspect to capture profile data (invocation count and aggregate time) per method.
 * The aspect is designed for load-time weaving using a concrete aspect subclass defined in an
 * {@code aop.xml} file. That concrete aspect must define a pointcut for the abstract {@link
 * #scope()} method.
 * <p>
 * Although the tracing advice implementation is designed to introduce as little overhead as
 * possible, the overhead is quite noticeable while data collection is active (less so while it
 * is disabled). This overhead will be reflected in the collected data, and it should be taken
 * into account when analyzing the results. Part of the overhead is due to the load-time weaving
 * of this aspect with the classes it is measuring. This weaving is done once, when the target
 * class is first loaded into the JVM. To minimize the impact of this one-time weave on the
 * trace results, it is recommended to perform the work flow to be measured once before starting
 * data collection, then perform it again with data collection turned on. However, this is not
 * always possible, if what is being measured is the first-time execution of a code path, where
 * future executions will be short-circuited by caching of various resources by the runtime.
 * <p>
 * Due to the aforementioned overhead which this feature introduces to running code, it is
 * recommended not to enable this feature in production deployments.
 * <p>
 * Tracing is controlled by the {@link #enableDataCollection(boolean)} method. When an active
 * tracing session is disabled, all data which has been collected up to that point is
 * post-processed and written to a CSV file, then discarded.
 */
@Aspect
public abstract class MethodTraceAspect
{
   /** Logger */
   private static final CentralLogger LOG = CentralLogger.get(MethodTraceAspect.class);
   
   static
   {
      if (LOG.isLoggable(Level.WARNING))
      {
         LOG.warning("Method profiling is enabled; " +
                     "this feature is not intended for production use!");
      }
   }
   
   /** The name of the folder in which the HTML UI artifacts will be generated */
   private static String TRACE_OUTPUT = "trace";
   
   /** The name of the folder where the trace data will be dumped to be used by the HTML UI */
   private static String DATA_OUTPUT = "data";
   
   /** Fixed name of CSV output file (written to current directory) */
   private static String CSV_FILE = "trace.csv";
   
   /** Fixed name of summary CSV output file (written to current directory) */
   private static String SUMMARY_FILE = "trace_summary.csv";
   
   /** Fixed name of all methods list CSV output file (written to current directory) */
   private static String METHOD_LIST_FILE = "trace_method_list.csv";
   
   /** Flag which controls data collection */
   private static volatile boolean enabled = false;
   
   /** Start time of a trace session, in milliseconds */
   private static long traceStart = 0L;
   
   /** Map of profile data by method signature */
   private static final Map<Call, TraceData> traceData = new ConcurrentHashMap<>();
   
   /** Thread-local stack of trace data since entering the first join point in the traced code */
   private static final ThreadLocal<Deque<TraceData>> callStack = new ThreadLocal<Deque<TraceData>>()
   {
      @Override
      public Deque<TraceData> initialValue()
      {
         return new ArrayDeque<>();
      }
   };
   
   /** Package "buckets" for data post-processing */
   private static final String[] buckets =
   {
      // TODO: hard-coded and duplicates information in aop.xml, but how to easily extract that?
      // When adding subpackages, write them below the existent parent package (if it exists).
      // This will make it easier to match the correct bucket
      "com.goldencode.p2j.persist.",
      "com.goldencode.p2j.persist.orm.",
      "com.mchange.",
      "org.postgresql.",
      "org.h2.",
   };
   
   /**
    * Enable or disable data collection. Upon disabling data collection, any collected results
    * are processed and exported.
    * 
    * @param   enable
    *          {@code true} to start data collection; {@code false} to stop it and process the
    *          results.
    */
   public synchronized static void enableDataCollection(boolean enable)
   {
      if (enable)
      {
         if (!enabled)
         {
            traceStart = System.currentTimeMillis();
            enabled = true;
         }
         
         return;
      }
      
      // stop collecting and process collected results
      
      try
      {
         // we potentially give up some accuracy in the results if a thread is finishing up its
         // collection work while we process the results, but this is done willingly to avoid
         // the overhead of synchronization while collecting trace data
         enabled = false;
         processData();
         exportHTML();
      }
      finally
      {
         // reset results
         traceData.clear();
         traceStart = 0L;
      }
   }
   
   /**
    * Routine used for generating a HTML UI used for better interpreting of the data. The end-user 
    * interface groups the method tracing on roots and buckets.
    */
   private static void exportHTML()
   {
      DecimalFormat df = new DecimalFormat("0.00000");
      df.setRoundingMode(RoundingMode.DOWN);
      long grandTotalElapsed = 0L;
      
      // organize trace data into buckets, indexed by entry point (first traced) method
      Map<MethodIdentifier, BucketData> bucketsByRoot = getBucketsByRoot();
      for (BucketData bucketData : bucketsByRoot.values())
      {
         grandTotalElapsed += bucketData.getElapsed();
      }
      
      /* Ensure the intermediary folders exist */
      new File(TRACE_OUTPUT + File.separator + DATA_OUTPUT).mkdirs();
      
      /* A meta file is used to track: the names of the buckets, the roots and the total elapsed time */
      String metaFile = TRACE_OUTPUT + File.separator + DATA_OUTPUT + File.separator + "meta.js";
      try (BufferedWriter metaWriter = new BufferedWriter(new FileWriter(metaFile)))
      {
         // total elapsed time
         metaWriter.append("var ").append("elapsed = " + grandTotalElapsed).append(";");
         metaWriter.newLine();
         // name of the buckets
         String bucketsArray = Arrays.asList(buckets).stream()
                  .map((value) -> "\"" + value + "\"").collect(Collectors.joining(", "));
         metaWriter.append("var ").append("buckets = ").append("[" + bucketsArray + "];");
         metaWriter.newLine();
         
         List<String> roots = new ArrayList<>();
         List<String> scriptInject = new ArrayList<>();
      
         for (Map.Entry<MethodIdentifier, BucketData> entry : bucketsByRoot.entrySet())
         {
            String root = entry.getKey().getShortName();
            BucketData data = entry.getValue();
            if (data.getElapsed() < 1000000)
            {
               // avoid exporting scenarios executed in < 1 ms
               continue;
            }

            // each root will have tracing information in a folder inside the data folder
            String rootFile = TRACE_OUTPUT + File.separator + DATA_OUTPUT + File.separator + root;
            new File(rootFile).mkdirs();
            roots.add(root);
            
            for (String bucket : buckets)
            {
               // keep names simple, without additional .
               // the decision to export each tracing information in separate JS is to ensure that
               // we can load the data statically (no need of file server to use the interface)
               String jsName = bucket.replace(".", "") + ".js";
               String bucketFile = rootFile + File.separator + jsName;
               try (BufferedWriter bucketWriter = new BufferedWriter(new FileWriter(bucketFile)))
               {
                  // traceData map is initialized in the loader; here we just add data for each bucket
                  bucketWriter.append("traceData[").append("\"" + root + "\"")  
                              .append("][").append("\"" + bucket + "\"").append("] = {");
                  long bucketElapsed = data.getTraces(bucket).getElapsed();
                  
                  // meta data like own_time, proc_time and grand_time
                  bucketWriter.append("\"own_time\" : ")
                              .append("" + bucketElapsed).append(", ");
                  bucketWriter.append("\"proc_time\" : ")
                              .append(df.format(bucketElapsed / (double) data.getElapsed())).append(", ");
                  bucketWriter.append("\"grand_time\" : ")
                              .append(df.format(bucketElapsed / (double) grandTotalElapsed)).append(", ");
                  
                  // method tracing is an array of objects
                  bucketWriter.append("\"traces\" : [");
                  bucketWriter.newLine();
                  for (TraceData trace : data.getTraces(bucket).getTraces())
                  {
                     bucketWriter.append("{");
                     MethodIdentifier callee = trace.callee;
                     bucketWriter.append("\"name\" : ");
                     if (callee != null)
                     {
                        Method method = trace.callee.method;
                        bucketWriter.append('"').append(method.getDeclaringClass().getName())
                                    .append('.').append(method.getName()).append('"');
                     }
                     else
                     {
                        bucketWriter.write("\"All others...\"");
                     }
                     bucketWriter.append(',').append("\"count\" : ")
                                 .append(Long.toString(trace.count.get()))
                                 .append(',').append("\"elapsed\" : ")
                                 .append(Long.toString(trace.elapsed.get()))
                                 .append(',').append("\"time_proc\" : ")
                                 .append(Double.toString(trace.elapsed.get() / data.getElapsed()))
                                 .append(',').append("\"owntime\" : ")
                                 .append(Long.toString(trace.getOwnTime()))
                                 .append(',').append("\"owntime_proc\" : ")
                                 .append(Double.toString(trace.getOwnTime() / data.getElapsed()))
                                 .append(',').append("\"grand_proc\" : ")
                                 .append(Double.toString(trace.getOwnTime() / (double) grandTotalElapsed));
                     bucketWriter.append("},");
                     bucketWriter.newLine();
                  }
                  
                  bucketWriter.append("]};");
               }
               
               // make sure we keep track of which js files we generates; these should be statically imported
               scriptInject.add(DATA_OUTPUT + File.separator + root + File.separator + jsName);
            }
         }
         // generate required loader (JS), run-time (JS) and bootstrap (CSS)
         generateAux(roots);
         String rootsArray = roots.stream()
                  .map((value) -> "\"" + value + "\"").collect(Collectors.joining(", "));
         metaWriter.append("var ").append("roots = ").append("[" + rootsArray + "];");
         
         // statically import each script directly from the HTML
         // if we use JS module import or fetch, we need a file server to satisfy CORS
         // we have control over HTML creation: just bump all JS there
         InputStream is = MethodTraceAspect.class.getResourceAsStream("web/method_tracing.html");
         String inputHtml = new BufferedReader(new InputStreamReader(is)).lines().collect(Collectors.joining("\n"));
         String extraScripts = scriptInject.stream()
                  .map((value) -> "<script type=\"text/javascript\" src=\"" + value + "\" defer></script>")
                  .collect(Collectors.joining("\n\t\t"));
         inputHtml = inputHtml.replace("<!-- Extra scripts -->", extraScripts);
         
         String outputHtmlPath = TRACE_OUTPUT + File.separator + "index.html";
         try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(outputHtmlPath)))
         {
            outputStream.write(inputHtml);
         }
      }
      catch (IOException e)
      {
         LOG.severe("", e);
      }
   }
   
   /**
    * Helper method for HTML UI generation. This ensures that the requires JS and CSS are generated
    * for the index HTML. Otherwise, functionality may lack.
    *  
    * @param   roots
    *          A list of all JS files we created containing information about multiple traced methods.
    *          These should be dumped in the loader JS which ensures that the data is loaded in the
    *          JS memory.
    */
   private static void generateAux(List<String> roots) 
   throws IOException
   {
      new File(TRACE_OUTPUT + File.separator + "js").mkdirs();
      new File(TRACE_OUTPUT + File.separator + "css").mkdirs();
      String loaderFile = TRACE_OUTPUT + File.separator + "js" + File.separator + "loader.js";
      try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(loaderFile)))
      {
         outputStream.write("var traceData = {};");
         outputStream.newLine();
         for (String root : roots)
         {
            outputStream.write("traceData[\"" + root + "\"] = {};");
            outputStream.newLine();
         }
      }
      
      String traceFile = TRACE_OUTPUT + File.separator + "js" + File.separator + "trace.js";
      InputStream is = MethodTraceAspect.class.getResourceAsStream("web/trace.js");
      String traceJS = new BufferedReader(new InputStreamReader(is)).lines().collect(Collectors.joining("\n"));
      try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(traceFile)))
      {
         outputStream.write(traceJS);
      }
   
      String bootstrapFile = TRACE_OUTPUT + File.separator + "css" + File.separator + "bootstrap.min.css";
      InputStream bis = MethodTraceAspect.class.getResourceAsStream("web/bootstrap.min.css");
      String bootstrapCss = new BufferedReader(new InputStreamReader(bis)).lines().collect(Collectors.joining("\n"));
      try (BufferedWriter outputStream = new BufferedWriter(new FileWriter(bootstrapFile)))
      {
         outputStream.write(bootstrapCss);
      }
      
   }
   
   /**
    * Utility method used by both HTML and CSV UI. This groups all tracked bucket data
    * per "root" (the method which determined the execution of all methods from the
    * bucket).
    * 
    * @return   A map of bucket data grouped by the root / scenario.
    */
   private static Map<MethodIdentifier, BucketData> getBucketsByRoot()
   {
      Map<MethodIdentifier, BucketData> bucketsByRoot = new HashMap<>();
      
      for (Map.Entry<Call, TraceData> e : traceData.entrySet())
      {
         Call call = e.getKey();
         TraceData trace = e.getValue();
         
         MethodIdentifier caller = call.caller;
         MethodIdentifier callee = call.callee;
         
         String callerBucket = matchBucket(caller);
         String calleeBucket = matchBucket(callee);
         
         boolean transition = !Objects.equals(callerBucket, calleeBucket);
         
         MethodIdentifier root = call.root;
         BucketData bucketData = bucketsByRoot.get(root);
         
         if (bucketData == null)
         {
            bucketData = new BucketData();
            bucketsByRoot.put(root, bucketData);
         }
         
         if (calleeBucket != null)
         {
            bucketData.addTrace(calleeBucket, trace, transition, root.equals(callee));
         }
      }
      return bucketsByRoot;
   }
   
   /**
    * Post-process the collected trace data and write the result to a CSV file. This involves
    * organizing the data into "buckets", each representing a transition from one code framework
    * to another. Only the trace data for the first method call in each new bucket is retained,
    * as it represents the aggregate time spent in that method and all of its called methods.
    * This reduced data set is then sorted in descending order by aggregate time spent in the
    * outermost method call across each set of buckets. This sorted data is then written out to
    * a CSV file.
    */
   private static void processData()
   {
      // get overall elapsed trace time in milliseconds.
      long elapsed = System.currentTimeMillis() - traceStart;
      
      // organize trace data into buckets, indexed by entry point (first traced) method
      
      Map<MethodIdentifier, BucketData> bucketsByRoot = getBucketsByRoot();
      
      // compute grand totals by bucket
      
      int len = buckets.length;
      
      long grandTotalElapsed = 0L;
      
      long[] totals = new long[len];
      
      for (BucketData bucketData : bucketsByRoot.values())
      {
         
         long across = 0;
         for (int i = 0; i < len; i++)
         {
            String bucket = buckets[i];
            TraceBucket traces = bucketData.getTraces(bucket);
            totals[i] += traces.getElapsed();
            across += traces.getElapsed();
         }
         
         // this is not ideal; the bucket data elapsed type can be sometimes different than the across time
         // as it includes the time spent in packages not tracked. across is always correct as it computes
         // the time spent in the profiled packages. However, this should be solved by adding all packages
         // needed by the tracker not to rely on across, but on the total time.
         if (across != bucketData.getElapsed())
         {
            grandTotalElapsed += across;
         }
         else
         {
            grandTotalElapsed += bucketData.getElapsed();
         }
      }
      
      // sort bucket groups by aggregate elapsed time of longest bucket
      SortedSet<BucketData> sortedBucketData = new TreeSet<>(new ByTotalTime());
      sortedBucketData.addAll(bucketsByRoot.values());
      
      // write trace data out to CSV, one column group per bucket
      
      try (BufferedWriter bw = new BufferedWriter(new FileWriter(CSV_FILE));
           BufferedWriter summary = new BufferedWriter(new FileWriter(SUMMARY_FILE));
           BufferedWriter allMethods = new BufferedWriter(new FileWriter(METHOD_LIST_FILE));)
      {
         // elapsed trace time
         bw.append("\"Elapsed trace time (ms):\",").append(Long.toString(elapsed)).append(",");
         summary.append("\"Elapsed trace time (ms):\",").append(Long.toString(elapsed)).append(",");
         bw.newLine();
         bw.newLine();
         summary.newLine();
         summary.newLine();
         
         summary.append('"').append("Root").append("\",");
         summary.append("\"Count\",").append("\"Agg Time (nano)\",").append("\"Own Time (nano)\",");
         
         // column headings
         for (String bucket : buckets)
         {
            String shortBucket = bucket.substring(0, bucket.length() - 1);
            bw.append('"').append(shortBucket).append("\",");
            bw.append("\"Count\",").append("\"Agg Time (nano)\",").append("\"% of Total\",");
            bw.append("\"Own Time (nano)\",").append("\"% of Total\",").append("\"% of Grand Total\",");

            summary.append('"').append(shortBucket + " (own time)").append("\",\"");
            summary.append(shortBucket + " (% of Total)").append("\",");
         }
         bw.newLine();
         summary.newLine();
         
         for (BucketData bucketData : sortedBucketData)
         {
            double totalElapsed = bucketData.getElapsed();
            
            for (TraceData[] row : bucketData.asRows())
            {
               for (int i = 0; i < len; i++)
               {
                  TraceData trace = row[i];
                  
                  if (trace == null)
                  {
                     bw.write("\"\",,,,,,,");
                     
                     continue;
                  }
                  
                  MethodIdentifier callee = trace.callee;
                  if (callee != null)
                  {
                     Method method = trace.callee.method;
                     bw.write('"');
                     bw.write(method.getDeclaringClass().getName());
                     bw.write('.');
                     bw.write(method.getName());
                     bw.write('"');
                  }
                  else
                  {
                     bw.write("\"All others...\"");
                  }
                  bw.write(',');
                  bw.write(Long.toString(trace.count.get()));
                  bw.write(',');
                  bw.write(Long.toString(trace.elapsed.get()));
                  bw.write(',');
                  bw.write(Double.toString(trace.elapsed.get() / totalElapsed));
                  bw.write(',');
                  // use getOwnTime; using directly the variable ownTime may result in 
                  // negative results, as the method tracing can close without finishing
                  // the own time computing
                  bw.write(Long.toString(trace.getOwnTime()));
                  bw.write(',');
                  bw.write(Double.toString(trace.getOwnTime() / totalElapsed));
                  bw.write(',');
                  bw.write(Double.toString(trace.getOwnTime() / (double) grandTotalElapsed));
                  bw.write(',');
               }
               
               bw.newLine();
            }
            
            // write statistics only about the root in the summary
            TraceData rootTrace = bucketData.root;
            if (rootTrace != null)
            {
               if (rootTrace.callee != null)
               {
                  Method rootMethod = rootTrace.callee.method;
                  summary.append('"').append(rootMethod.getDeclaringClass().getName());
                  summary.append('.').append(rootMethod.getName()).append("\",");
               }
               else
               {
                  bw.write("\"Other...\"");
               }
               summary.append(Long.toString(rootTrace.count.get())).append(",");
               summary.append(Long.toString(rootTrace.elapsed.get())).append(",");
               summary.append(Long.toString(rootTrace.getOwnTime())).append(",");
            }
            else
            {
               summary.append(",,,,");
            }
            
            // write totals
            Iterator<TraceBucket> iter = bucketData.traces();
            while (iter.hasNext())
            {
               TraceBucket bucket = iter.next();
               
               long bucketElapsed = bucket.getElapsed();
               
               bw.write("\"   Bucket Totals...\",,,,");
               summary.append(Long.toString(bucketElapsed)).append(",");
               summary.append(Double.toString(bucketElapsed / totalElapsed)).append(",");
               bw.write(Long.toString(bucketElapsed));
               bw.write(',');
               bw.write(Double.toString(bucketElapsed / totalElapsed));
               bw.write(',');
               bw.write(Double.toString(bucketElapsed / (double) grandTotalElapsed));
               bw.write(',');
            }
            
            bw.newLine();
            bw.newLine();
            summary.newLine();
         }
         
         writeAllMethods(allMethods);
         
         // write grand totals
         writeGrandTotals(bw, len, totals, grandTotalElapsed);
         writeGrandTotals(summary, len, totals, grandTotalElapsed);
      }
      catch (IOException exc)
      {
         LOG.log(Level.SEVERE, "I/O error writing trace data CSV", exc);
      }
   }
   
   /**
    * Utility method for dumping all the methods traces together with count and own time.
    * 
    * @param   bw
    *          The writter where this statistic should be written.
    */
   private static void writeAllMethods(BufferedWriter bw) 
   throws IOException
   {      
      Map<MethodIdentifier, TraceData> dataPerMethod = aggregatePerMethod();
      
      bw.append("\"Package\",\"Class\",\"Method name\",\"Count\",\"Own Time\"");
      bw.newLine();
      
      List<String> lines = new ArrayList<>();
      for (Map.Entry<MethodIdentifier, TraceData> entry : dataPerMethod.entrySet())
      {
         StringBuilder sb = new StringBuilder();
         String className = entry.getKey().method.getDeclaringClass().getName();
         sb.append('"').append(className.substring(0, className.lastIndexOf(".")));
         sb.append("\",");
         sb.append('"').append(className).append("\",");
         sb.append('"').append(className);
         sb.append('.').append(entry.getKey().method.getName());
         Class<?>[] params = entry.getKey().method.getParameterTypes();
         sb.append("(");
         for (int i = 0; i < params.length; i++)
         {
            if (i != 0)
            {
               sb.append(",");
            }
            sb.append(params[i].getSimpleName());
         }
         sb.append(")").append("\",");
         
         sb.append(Long.toString(entry.getValue().count.get())).append(",");
         sb.append(Long.toString(entry.getValue().getOwnTime())).append(",");
         lines.add(sb.toString());
      }
      
      Collections.sort(lines);

      for (String line : lines)
      {
         bw.append(line);
         bw.newLine();
      }
      
      bw.newLine();
   }
   
   /**
    * Retrieve a map between each tracked method and its trace data. This is an aggregate
    * of all buckets. A method can appear in multiple buckets - here we compute the
    * total time. Don't use the caller of the trace data as there is none - a trace
    * data holds information about multiple traces. Its own time and count is in fact
    * a total of all traces with the same method identifier.
    * 
    * @return   A map between all methods and their total own time and count across
    *           all buckets.
    */
   private static Map<MethodIdentifier, TraceData> aggregatePerMethod()
   {
      Map<MethodIdentifier, TraceData> dataByMethod = new HashMap<>();
      
      for (Map.Entry<Call, TraceData> e : traceData.entrySet())
      {
         Call call = e.getKey();
         TraceData trace = e.getValue();
         
         MethodIdentifier callee = call.callee;
         TraceData methodData = dataByMethod.get(callee);
         
         if (methodData == null)
         {
            methodData = new TraceData(null, callee);
            dataByMethod.put(callee, methodData);
         }
         methodData.count.addAndGet(trace.count.get());
         // use getOwnTime to avoid negative results; this can happen when the tracing stops
         // before finishing the computation of own time
         methodData.ownTime.addAndGet(trace.getOwnTime());
      }
      return dataByMethod;
   }
   
   /**
    * Utility method for CSV exporting of the data. More exactly, this prints the grand totals
    * pe bucket fort he active method tracing.
    * 
    * @param   bw
    *          The writer where the grand totals shoudl be written
    * @param   len
    *          The number of buckets
    * @param   totals
    *          The total values
    * @param   grandTotalElapsed
    *          The grand total elapsed
    */
   private static void writeGrandTotals(BufferedWriter bw, int len, long[] totals, long grandTotalElapsed) 
   throws IOException
   {
      bw.newLine();
      
      bw.write("\"Grand Totals\",");
      bw.newLine();
      bw.write("\"Bucket\",,\"Agg Own Time (nano)\",\"% of Grand Total\"");
      bw.newLine();
      
      for (int i = 0; i < len; i++)
      {
         bw.append('"').append(buckets[i]).append("\",,").append(Long.toString(totals[i]));
         bw.append(',').append(Double.toString(totals[i] / (double) grandTotalElapsed)).append(',');
         bw.newLine();
      }
   }
   
   /**
    * Attempt to match a method's declaring class with a data bucket identifier.
    * 
    * @param   ident
    *          Method identifier.
    * 
    * @return  The matching bucket identifer string, if a match was found, else {@code null}.
    */
   private static String matchBucket(MethodIdentifier ident)
   {
      Method method = ident.method;
      
      if (method == null)
      {
         return null;
      }
      
      String name = method.getDeclaringClass().getName();
      String bestMatch = null;
      for (String bucket : buckets)
      {
         if (name.startsWith(bucket))
         {
            // this depends on the fact that all packages and subpackages from buckets
            // are placed in the correct order
            bestMatch = bucket;
         }
      }
      
      return bestMatch;
   }
   
   /**
    * An abstract pointcut which should be defined for load-time weaving inside a {@code
    * concrete-aspect} element within {@code META-INF/aop.xml}.
    */
   @Pointcut
   public abstract void scope();
   
   /**
    * The primary tracing advice which collects trace information about calls to the target
    * methods. Here we attempt the following with minimal overhead:
    * <ul>
    * <li>Determine the caller method.
    * <li>Determine the callee method.
    * <li>Measure the elapsed number of nanoseconds it takes for the callee method to execute
    *     and add that value to a running aggregate for this method and subtract it from the
    *     own time of our calling method, if any.
    * <li>Increment the number of times this caller invoked this callee.
    * </ul>
    * <p>
    * Data is collected safely across all threads on which calls to the traced methods are made.
    * <p>
    * The caller method is determined using a thread-local stack, which is cleared any time this
    * advice is entered after data collection has been halted. Note that the relationship between
    * caller and callee may represent an actual, direct invocation if both methods are crosscut
    * by the defined pointcut. Otherwise, if intermediate method invocations are not crosscut,
    * the recorded "caller" may actually represent some "ancestor" method further up the actual
    * call stack. It is assumed that this potential "gap" between caller and callee is understood
    * by and acceptable to the definer of the pointcut.
    * 
    * @param   joinPoint
    *          Join point which represents the method being traced.
    *          
    * @return  Object, if any, returned by the method being traced.
    * 
    * @throws  Throwable
    *          if an exception is thrown by the method being traced.
    */
   @Around("scope()")
   public Object trace(ProceedingJoinPoint joinPoint)
   throws Throwable
   {
      if (!enabled)
      {
         // clean up any work in process from earlier tracing session
         callStack.remove();
         
         return joinPoint.proceed();
      }
      
      Deque<TraceData> stack = callStack.get();
      boolean outer = stack.isEmpty();
      
      JoinPoint.StaticPart staticPart = joinPoint.getStaticPart();
      MethodSignature sig = (MethodSignature) staticPart.getSignature();
      
      TraceData parent = outer ? null : stack.peek();
      MethodIdentifier caller = outer ? MethodIdentifier.UNTRACED : parent.callee;
      MethodIdentifier callee = new MethodIdentifier(sig.getMethod(), staticPart.getId());
      MethodIdentifier root = outer ? callee : stack.peekLast().callee;
      
      // path from root to callee (these are the same if this is the first traced call)
      Call path = new Call(root, caller, callee);
      
      // look up the aggregate trace data for this combination of caller and callee, if any such
      // information already has been initialized; if not found, initialize it
      TraceData data = traceData.get(path);
      if (data == null)
      {
         data = new TraceData(caller, callee);
         TraceData existing = traceData.putIfAbsent(path, data);
         if (existing != null)
         {
            data = existing;
         }
      }
      
      // emulate this thread's call stack, so nested calls have access to calling context
      stack.push(data);
      
      Object returnObject;
      long start = System.nanoTime();
      
      try
      {
         returnObject = joinPoint.proceed();
      }
      finally
      {
         // update invocation count and aggregate elapsed time for this call
         long elapsed = System.nanoTime() - start;
         
         if (enabled)
         {
            data.count.incrementAndGet();
            data.elapsed.addAndGet(elapsed);
            data.ownTime.addAndGet(elapsed);
            
            if (parent != null)
            {
               // subtract my elapsed time from my parent's own time
               // NOTE that this happens only when the tracing is enabled. This can
               // remain negative at the end of the tracing; use getOwnTime when enabled is false
               parent.ownTime.addAndGet(-elapsed);
            }
            
            stack.pop();
         }
         else
         {
            callStack.remove();
         }
      }
      
      return returnObject;
   }
   
   /**
    * A wrapper for a method, optimized for use as a hash map key. {@code
    * java.lang.reflect.Method} is not used directly, because its {@code hashCode()}
    * implementation operates on string data, making it slow as a hash map key.
    */
   private static class MethodIdentifier
   {
      /** Singleton instance meant to represent an untraced method */
      static final MethodIdentifier UNTRACED = new MethodIdentifier();
      
      /** The wrapped method */
      private final Method method;
      
      /** Identifier assigned by AspectJ, unique within the method's class */
      private final int id;
      
      /** Cached hash code */
      private final int hash;
      
      /**
       * Constructor which computes and caches hash code.
       * 
       * @param   method
       *          The wrapped method.
       * @param   id
       *          Identifier assigned by AspectJ, unique within the method's class.
       */
      MethodIdentifier(Method method, int id)
      {
         this.method = method;
         this.id = id;
         
         // we calculate hash using the method's class because Method's hashCode() implementation
         // seems pretty inefficient; Class just inherits Object.hashCode()
         // the jointpoint's static part id already gives us a unique identifier for a method
         // within a type
         Class<?> type = method.getDeclaringClass();
         int result = 17;
         result = 37 * result + id;
         this.hash = 37 * result + type.hashCode();
      }
      
      /**
       * Private default constructor intended only to create UNKNOWN singleton.
       */
      private MethodIdentifier()
      {
         this.method = null;
         this.hash = 0;
         this.id = -1;
      }
      
      /**
       * Fast implementation which returns the cached hash code.
       * 
       * @return  Hash code.
       */
      @Override
      public final int hashCode()
      {
         return hash;
      }
      
      /**
       * Implementation which is consistent with overridden {@code hashCode()} method.
       * 
       * @param   o
       *          Object with which to compare this instance.
       * 
       * @return  {@code true} if the given instance is equivalent to this instance.
       */
      @Override
      public final boolean equals(Object o)
      {
         MethodIdentifier that = (MethodIdentifier) o;
         
         // quick out
         if (this.id != that.id)
         {
            return false;
         }
         
         if (this == UNTRACED || that == UNTRACED)
         {
            // should only be one instance of UNKNOWN
            return this == that;
         }
         
         // we use Class.equals() to be consistent with our implementation of hashCode
         Class<?> thisType = this.method.getDeclaringClass();
         Class<?> thatType = that.method.getDeclaringClass();
         
         // short-circuit, direct comparison is safe, since Class does not override
         // Object.equals(), which just uses reference identity
         return thisType == thatType;
      }
      
      /**
       * Compose a string representation of this object, primarily for debugging.
       * 
       * @return  String representation of this object.
       */
      @Override
      public String toString()
      {
         return (method == null ? "UNKNOWN" : method.toString()) + " <" + id + ">";
      }
      
      /**
       * Retrieve a simple, yet not that descriptive name. This includes only
       * the class name, without packages.
       * 
       * @return   A shorter representative name.
       */
      public String getShortName()
      {
         return method.getDeclaringClass().getSimpleName().replace(".", "").replace("$", "") + 
                "_" + method.getName() + "_" + id;
      }
   }
   
   /**
    * Representation of a method invocation which identifies the caller and the callee methods,
    * optimized for use as a hash map key.
    */
   private static class Call
   {
      /** First method in call path traced */
      private final MethodIdentifier root;
      
      /** Caller method */
      private final MethodIdentifier caller;
      
      /** Callee method */
      private final MethodIdentifier callee;
      
      /** Cached hash */
      private final int hash;
      
      /**
       * Constructor which computes and caches hash code.
       * 
       * @param   root
       *          First method in call path traced.
       * @param   caller
       *          Caller method.
       * @param   callee
       *          Callee method.
       */
      Call(MethodIdentifier root, MethodIdentifier caller, MethodIdentifier callee)
      {
         this.root = root;
         this.caller = caller;
         this.callee = callee;
         
         int result = 17;
         result = 37 * result + root.hashCode();
         result = 37 * result + caller.hashCode();
         this.hash = 37 * result + callee.hashCode();
      }
      
      /**
       * Fast implementation which returns the cached hash code.
       * 
       * @return  Hash code.
       */
      @Override
      public int hashCode()
      {
         return hash;
      }
      
      /**
       * Implementation which is consistent with overridden {@code hashCode()} method.
       * 
       * @param   o
       *          Object with which to compare this instance.
       * 
       * @return  {@code true} if the given instance is equivalent to this instance.
       */
      @Override
      public boolean equals(Object o)
      {
         Call that = (Call) o;
         
         return this.root.equals(that.root) &&
                this.caller.equals(that.caller) &&
                this.callee.equals(that.callee);
      }
      
      /**
       * Compose a string representation of this object, primarily for debugging.
       * 
       * @return  String representation of this object.
       */
      @Override
      public String toString()
      {
         return root.toString() + "..." + System.lineSeparator() +
                "   " + caller + " ---> " + callee;
      }
   }
   
   /**
    * An interface with one method to return elapsed time.
    */
   private static interface ElapsedTimeable
   {
      /**
       * Get elapsed time.
       * 
       * @return  Elapsed time.
       */
      public long getElapsed();
   }
   
   /**
    * Data structure containing invocation count and aggregate elapsed time for a method call.
    * A method call implies a specific combination of caller method and callee method, but only
    * the callee method is stored within this class.
    * <p>
    * The numeric fields which track invocation count and elapsed time are intended to be
    * accessed directly by outside code, potentially on different threads. Updates to these
    * fields individually are guaranteed to be atomic, but updates to all fields are not. Thus,
    * there is a small window of opportunity for some of these fields and not others to be
    * updated as data collection ends and the data is post-processed. This may introduce a small
    * inconsistency in the data. However, given the purpose of profiling to get an overview of
    * many method invocations, this potential for minor inconsistency is deemed acceptable, in
    * order to avoid the overhead of thread synchronization during data collection.
    */
   private static class TraceData
   implements ElapsedTimeable
   {
      /** Number of times method has been invoked */
      final AtomicLong count = new AtomicLong(0);
      
      /** Aggregate elapsed time spent in method (includes aspect overhead and called methods) */
      final AtomicLong elapsed = new AtomicLong(0);
      
      /** Aggregate own time spent in method, not including called (traced) methods */
      final AtomicLong ownTime = new AtomicLong(0);
      
      /** The invoking method */
      private final MethodIdentifier caller;
      
      /** The invoked method */
      private final MethodIdentifier callee;
      
      /**
       * Constructor which accepts an identifier of the invoked method.
       * 
       * @param   caller
       *          Invoking method.
       * @param   callee
       *          Invoked method.
       */
      TraceData(MethodIdentifier caller, MethodIdentifier callee)
      {
         this.caller = caller;
         this.callee = callee;
      }
      
      /**
       * Get the aggregate elapsed time for all method calls held by this trace data.
       * 
       * @return  Aggregate elapsed time.
       */
      @Override
      public long getElapsed()
      {
         return elapsed.get();
      }
      
      /** 
       * Retrieve the own time in a safe manner. Note that own time can be negative 
       * as tracing can be closed before adding the total time to a method. The sub-calls
       * are subtracting their agg time from parent's own time, so methods which do not
       * end (due to tracing close) are having a negative own time. Which an error margin,
       * we can presume that the own time is 0, as the agg time is spent only in 
       * sub-calls.
       * 
       * @return   The own time of the trace data or {@code 0} if the own time is negative.
       */
      public long getOwnTime()
      {
         long snap = ownTime.get();
         return snap < 0 ? 0 : snap;
      }
      
      /**
       * Debug string representation.
       * 
       * @return  Debug information.
       */
      public String toString()
      {
         double msElapsed = elapsed.get() / 1000000.0;
         double msOwnTime = getOwnTime() / 1000000.0;
         long c = count.get();
         double avgElapsed = msElapsed / c;
         double avgOwnTime = msOwnTime / c;
         
         return "invoked: " + c +
                "; elapsed: " + msElapsed +
                "ms; own time: " + msOwnTime +
                " ms; avg elapsed: " + avgElapsed +
                " ms; avg own time: " + avgOwnTime + "ms" +
                callee != null ? ("; " + caller.method + " ---> " + callee.method) : "";
      }
   }
   
   /**
    * A comparator which sorts trace data objects into descending order by total elapsed time.
    */
   private static class ByTotalTime
   implements Comparator<ElapsedTimeable>
   {
      @Override
      public int compare(ElapsedTimeable o1, ElapsedTimeable o2)
      {
         long diff = o1.getElapsed() - o2.getElapsed();
         
         return diff == 0L ? 0 : diff < 0L ? 1 : -1;
      }
   }
   
   /**
    * A class which organizes method trace data into specific buckets.
    */
   private static class BucketData
   implements ElapsedTimeable
   {
      /** Map of bucket names to buckets of trace data */
      private final Map<String, TraceBucket> bucketData = new LinkedHashMap<>();
      
      /** Elapsed time of the root method call, including all its nested calls */
      private long elapsed = 0L;
      
      /** The root which determined this bucket */
      private TraceData root = null;
      
      /**
       * Constructor.
       */
      BucketData()
      {
         // prime the map with empty trace data sets
         for (String bucket : buckets)
         {
            bucketData.put(bucket, new TraceBucket());
         }
      }
      
      /**
       * Add aggregated trace data for all calls to a particular method to the given bucket.
       * 
       * @param   bucket
       *          Name of bucket.
       * @param   trace
       *          Aggregated trace data for a method.
       * @param   transition
       *          {@code true} if the method call captured by this trace represents a call into
       *          this bucket from another or from an untraced caller, else {@code false}.
       * @param   isRoot
       *          {@code true} if this trace represents the root (first traced) method, else
       *          {@code false}. A root method may call zero or more other traced methods.
       *          There can be only one (and there must be one) root trace per bucket data
       *          instance. Any other traces represent nested method calls in a graph originating
       *          from the root method.
       */
      void addTrace(String bucket, TraceData trace, boolean transition, boolean isRoot)
      {
         TraceBucket traces = bucketData.get(bucket);
         
         traces.addTrace(trace, transition);
         
         if (isRoot)
         {
            this.root = trace;
            elapsed = trace.elapsed.get();
         }
      }
      
      /**
       * Get the elapsed time of the longest bucket.
       * 
       * @return  Elapsed time of the longest bucket, in nanoseconds.
       */
      @Override
      public long getElapsed()
      {
         return elapsed;
      }
      
      /**
       * Get the trace data for the given bucket.
       * 
       * @param   bucket
       *          Name of bucket.
       * 
       * @return  Trace data associated with the given bucket.
       */
      TraceBucket getTraces(String bucket)
      {
         return bucketData.get(bucket);
      }
      
      /**
       * Get an iterator on the trace buckets.
       * 
       * @return  Trace bucket iterator.
       */
      Iterator<TraceBucket> traces()
      {
         return bucketData.values().iterator();
      }
      
      /**
       * Return all the stored traces across buckets as rows. Each row is an array of {@code
       * TraceData} objects. The number of elements in the array matches the number of buckets
       * into which the data is being organized. Since the buckets may contain a varying number
       * of traces, {@code null} is inserted into the array in place of a missing trace.
       * <p>
       * There is no implied, direct caller to callee relationship between the methods across the
       * columns of any particular row.
       * 
       * @return  A list of {@code TraceData} arrays. The number of rows matches the number of
       *          traces in the largest bucket.
       */
      List<TraceData[]> asRows()
      {
         List<Iterator<TraceData>> iters = new ArrayList<>();
         for (TraceBucket traces : bucketData.values())
         {
            iters.add(traces.getTraces().iterator());
         }
         
         int len = bucketData.size();
         
         List<TraceData[]> rows = new ArrayList<>();
         
         while (true)
         {
            boolean anyHasNext = false;
            
            for (Iterator<TraceData> iter : iters)
            {
               if (iter.hasNext())
               {
                  // the buckets may have a varying number of traces; keep going while any
                  // bucket has more traces
                  anyHasNext = true;
               }
            }
            
            if (!anyHasNext)
            {
               break;
            }
            
            TraceData[] row = new TraceData[len];
            rows.add(row);
            
            for (int i = 0; i < len; i++)
            {
               Iterator<TraceData> iter = iters.get(i);
               row[i] = iter.hasNext() ? iter.next() : null;
            }
         }
         
         return rows;
      }
   }
   
   /**
    * Trace data for a single analysis bucket, including the trace data itself and an aggregate
    * of all the traces' own times.
    */
   private static class TraceBucket
   {
      /** Trace data sorted in descendig order by full elapsed time of each call */
      private final SortedSet<TraceData> traces = new TreeSet<>(new ByTotalTime());
      
      /** Elapsed time spent net of called (traced) methods (own time) */
      private long elapsed = 0L;
      
      /** Trace data object representing all non-transition trace data */
      private TraceData allOthers = null;
      
      /**
       * Add a trace and add its own time to the aggregate own time for this analysis bucket.
       * 
       * @param   trace
       *          Trace data for a method call.
       * @param   transition
       *          {@code true} if the method call captured by this trace represents a call into
       *          this bucket from another or from an untraced caller, else {@code false}.
       */
      void addTrace(TraceData trace, boolean transition)
      {
         if (transition)
         {
            traces.add(trace);
         }
         else
         {
            if (allOthers == null)
            {
               allOthers = new TraceData(null, null);
               traces.add(allOthers);
            }
            
            // use getOwnTime; if trace was not closed by the tracing, the 
            // real ownTime may be negative. This is not of interest here
            allOthers.ownTime.addAndGet(trace.getOwnTime());
         }
         
         elapsed += trace.getOwnTime();
      }
      
      /**
       * Get the set of traces in this analysis bucket, sorted in descending order by full,
       * elapsed times.
       * 
       * @return  All trace data in this bucket.
       */
      SortedSet<TraceData> getTraces()
      {
         return traces;
      }
      
      /**
       * Get the aggregate own times of all traces in this analysis bucket.
       * 
       * @return  Aggregate of all traces' own times.
       */
      long getElapsed()
      {
         return elapsed;
      }
   }
}