# CallGraphApi.java

    /*
    ** Module   : CallGraphApi.java
    ** Abstract : Provides an interface for querying, traversing and manipulating the call graph
    **            from a web client.
    **
    ** Copyright (c) 2017-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------Description----------------------------------
    ** 001 GES 20170518 First version.
    ** 002 CA  20180320 Ambiguous report now lists the hint ID.
    **                  A performance improvement for the dependency report.
    **     CA  20180326 Some improvements for the Dead Code and Dependencies reports.
    ** 003 CA  20180413 Performance improvement: during report generation, some parents may be cached
    **                  in the vertex properties; if so, we need to commit the tx.
    **     CA  20180418 Re-write reports to improve performance.
    **                  Dead code now reports all files with possible dead entry points.
    **     CA  20180508 Ambiguous Call Sites report shows only processed (finished) files.
    **     CA  20180509 Allow include-level hints to disambiguate call sites.
    **     CA  20180525 Do not include virtual funcs/procs in the dead code details for a program file.
    ** 004 CA  20181003 Expose all non-virtual entry points in the callgraph.
    ** 005 CA  20200428 Some changes to process DEFINE ENUM.  Untested.
    ** 006 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.report.server;

    import java.io.*;
    import java.util.*;
    import java.util.concurrent.atomic.*;
    import java.util.function.*;
    import java.util.logging.*;

    import com.goldencode.p2j.cfg.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.uast.*;

    import org.apache.tinkerpop.gremlin.structure.*;

    import static org.apache.tinkerpop.gremlin.process.traversal.P.*;

    /**
     * Provides an interface for querying, traversing and manipulating the call graph from a
     * web client.
     */
    public class CallGraphApi
    implements ProgressParserTokenTypes
    {
       /** Traverse the graph in both upstream and downstream directions. */
       public static int BOTH_DIRECTIONS = 0;

       /** Traverse the graph in the downstream direction. */
       public static int DOWNSTREAM = 1;

       /** Traverse the graph in the upstream direction. */
       public static int UPSTREAM = 2;

       /** Logger. */
       private static final ConversionStatus LOG = ConversionStatus.get(CallGraphApi.class);

       /** Call graph API base message ID. */
       private static final int MSG_BASE = ReportProtocol.MSG_BASE + 2000;

       /** Message ID for reading the graph. */
       private static final int MSG_READ_GRAPH = MSG_BASE + 1;

       /** Message ID for missing targets report. */
       private static final int MSG_REPORT_MISSING_TARGETS = MSG_BASE + 2;

       /** Message ID for call-sites right-side report. */
       private static final int MST_REPORT_CALL_SITES = MSG_BASE + 3;

       /** Message ID for external dependencies report. */
       private static final int MST_REPORT_EXTERNAL_DEP = MSG_BASE + 4;

       /** Message ID for dead code report. */
       private static final int MST_REPORT_DEAD_CODE = MSG_BASE + 5;

       /** Message ID for ambiguous call-sites report. */
       private static final int MST_REPORT_AMBIG_CALLSITES = MSG_BASE + 6;

       /** Message ID for right-side ambiguous file's call-sites report. */
       private static final int MST_REPORT_FILE_AMBIG_CALLSITES = MSG_BASE + 7;

       /** Message ID for right-side ambiguous file's call-sites report. */
       private static final int MST_REPORT_DEAD_DEPENDENCIES = MSG_BASE + 8;

       /** Message ID for verify schema triggers report. */
       private static final int MST_REPORT_VERIFY_SHEMA_TRIGGERS = MSG_BASE + 9;

       /** Message ID for dependencies report. */
       private static final int MST_REPORT_LIST_DEPENDENCIES  = MSG_BASE + 10;

       /** Message ID for right-side dependencies report. */
       private static final int MST_REPORT_DEPENDENCIES_FOR = MSG_BASE + 11;

       /** The callgraph is emitted only for file resources. */
       private static final int GRAPH_DETAIL_FILES = 0;

       /** The callgraph is emitted for only for entry points. */
       private static final int GRAPH_DETAIL_ENTRY_POINTS = 1;

       /** The callgraph is emitted with all details. */
       private static final int GRAPH_DETAIL_ALL = 2;

       /** Graph database instance to use. */
       private static Graph db = CallGraphHelper.getGraph();

       /**
        * Default constructor.
        */
       public CallGraphApi()
       {
       }

       /**
        * Build the {@link CallSite} instance with the details for the specified callgraph vertex.
        *
        * @param    v
        *           The callgraph vertex object.
        *
        * @return   The {@link CallSite} instance.
        */
       public static CallSite buildCallSite(Vertex v)
       {
          Vertex vfile = null;
          Vertex vAst = null;
          String filename = null;

          int srcType = ReportApi.SRC_CODE_CACHE;

          if (v.<Integer>property("resource-domain").value() == AST_NODE ||
              v.<Integer>property("resource-domain").value() == FILE_RESOURCE)
          {
             vfile = CallGraphHelper.findProcedureFile(v);

             if (vfile == null)
             {
                vfile = CallGraphHelper.findSchemaFile(v);
                srcType = ReportApi.SRC_SCHEMA;

                if (vfile != null)
                {
                   vAst = db.traversal().V(vfile).out("contains")
                            .has("node-type", DATABASE).next();

                   filename = vfile.<String>property("srcfile").value();
                }
             }
             else
             {
                filename = vfile.<String>property("os-filename").value();
                vAst = db.traversal().V(vfile).out("contains")
                         .has("node-type", EXTERNAL_PROCEDURE).next();
             }
          }

          if (vfile != null)
          {
             // this is the root node of the file, NOT the actual call site's AST id
             long astId = vAst.<Long>property("node-id").value();

             CallSite cs = new CallSite();
             cs.setFilename(filename);
             cs.setLine(v.property("line").isPresent() ? v.<Integer>property("line").value() : 0);
             cs.setColumn(v.property("column").isPresent() ? v.<Integer>property("column").value() : 0);
             cs.setVertexId(v.id().toString());
             cs.setCallSite(describeNode(v, v.<Integer>property("node-type").value()));
             cs.setAstId(astId);
             cs.setNodeId(v.<Long>property("node-id").value());
             cs.setTokenType(v.property("type").isPresent() ? v.<Integer>property("type").value() : 0);
             cs.setTokenText(v.property("text").isPresent() ? v.<String>property("text").value() : "");
             cs.setSrcType(srcType);
             try
             {
                cs.setFid(ReportApi.getFileId(filename));
             }
             catch (Throwable t)
             {
                // ignore
             }

             return cs;
          }

          return null;
       }

       /**
        * Calculate the descriptive text for a given vertex.
        *
        * @param    v
        *           The vertex to describe.
        * @param    type
        *           The token type for the node.
        *
        * @return   The descriptive text for the node.
        */
       private static String describeNode(Vertex v, int type)
       {
          String label = null;
          int astType = (v.property("type").isPresent() ? v.<Integer>property("type").value() : 0);

          if (type == EXTERNAL_PROCEDURE ||
              astType == KW_RUN          ||
              astType == KW_ASYNC        ||
              astType == COM_METHOD      ||
              (astType > BEGIN_METH && astType < END_METH) ||
              (astType > BEGIN_FUNCTYPES && astType < END_FUNCTYPES))
          {
             label = ProgressParser.lookupTokenName(type);
          }
          else if (v.property("os-filename").isPresent())
          {
             label = (String) CallGraphHelper.property(v, "os-filename");
          }
          else if (v.property("node-key").isPresent())
          {
             label = CallGraphHelper.getNodeKeyValue(v);
          }
          else
          {
             label = v.label();
          }

          return label;
       }

       /**
        * Calculate the descriptive text for a given vertex.
        *
        * @param    v
        *           The vertex to describe.
        * @param    type
        *           The token type for the node.
        *
        * @return   The descriptive text for the node.
        */
       private static String describeTooltip(Vertex v, int type)
       {
          String tooltip = null;

          int astType = (v.property("type").isPresent() ? v.<Integer>property("type").value() : 0);

          if (astType == KW_ON)
          {
             String events = v.property("events").isPresent() ? v.<String>property("events").value()
                                                              : null;

             if (events != null)
             {
                tooltip = events;
             }

             return tooltip;
          }

          // only these have custom tooltip for now
          if (!(astType == KW_RUN       ||
                type == EVENT           ||
                astType == KW_ASYNC     ||
                type == FUNCTION_CALL   ||
                type == KW_DYN_FUNC))
          {
             return null;
          }

          tooltip = "";

          Iterator<Vertex> iter = db.traversal().V(v).out("calls");
          while (iter.hasNext())
          {
             Vertex targetV = iter.next();
             int nodeType = targetV.<Integer>property("node-type").value();
             Vertex vFile = null;

             int vDom = targetV.<Integer>property("resource-domain").value();
             if (vDom == AST_NODE)
             {
                vFile = CallGraphHelper.findProcedureFile(targetV);
             }

             String txt = "";
             if (vFile != null && !vFile.equals(targetV))
             {
                 txt += vFile.<String>property("filename").value();
             }

             if (nodeType != EXTERNAL_PROCEDURE)
             {
                if (targetV.<String>property("node-key").isPresent())
                {
                   txt += " " + CallGraphHelper.getNodeKeyValue(targetV);
                }
                else
                {
                   txt += " " + targetV.label();
                }
             }

             if (targetV.property("line").isPresent())
             {
                int line = targetV.<Integer>property("line").value();
                int column = targetV.<Integer>property("column").value();
                String lbl = nodeType == TRIGGER_BLOCK ? "registers" : "contains";

                Vertex parentV = targetV;
                while (parentV != null && line == 0 && column == 0)
                {
                   Iterator<Vertex> iterP = db.traversal().V(parentV).in(lbl);
                   if (iterP.hasNext())
                   {
                      parentV = iterP.next();
                      if (parentV.property("line").isPresent())
                      {
                         line = parentV.<Integer>property("line").value();
                         column = parentV.<Integer>property("column").value();
                      }
                   }
                   else
                   {
                      parentV = null;
                   }
                }

                if (parentV != null && line != 0 && column != 0)
                {
                   txt += "@" + line + ":" + column;
                }
             }

             if (!tooltip.isEmpty())
             {
                tooltip += "; ";
             }

             tooltip += txt;
          }

          tooltip = tooltip.trim();
          return tooltip.isEmpty() ? null
                                   : tooltip.indexOf(";") == -1 ? tooltip
                                                                : " Calls: " + tooltip.trim();
       }

       /**
        * Obtain a subset of the call graph based on the given criteria.
        *
        * @param    files
        *           The list of AST filenames to use as the starting point for the graph snippet
        *           or <code>null</code> to use the project's preconfigured root node list.
        * @param    profileId
        *           If greater than 0, then is the ID of a filter profile.
        * @param    direction
        *           One of the direction constants from this class for {@link #BOTH_DIRECTIONS},
        *           {@link #DOWNSTREAM} or {@link #UPSTREAM}. This controls which edges (inbound
        *           and/or outbound) are included.  UPSTREAM maps to inbound and DOWNSTREAM maps
        *           to outbound.
        * @param    levels
        *           This defines the number of cross-file levels of traversal to include in the
        *           snippet. Since some nodes are contained (using edges labeled "contains") in
        *           other nodes, using a simple calculation of edge levels would lead to inconsistent
        *           results.  Instead, we track the number of times that the graph traveral crosses
        *           from one file domain resource into another file domain resource. Use 0 to
        *           include only the starting nodes, 1 to include the starting nodes plus the nodes
        *           that can be traversed through 1 edge (in the direction specified), 2 for
        *           traversal through 2 edges and so forth. -1 means there is no limit. Use the no
        *           limit options with caution as there may be performance and memory issues with
        *           the result.
        * @param    detail
        *           The level of detail for the graph.
        *
        * @return   The specified subset of the graph.
        */
       @WebApi(type = MSG_READ_GRAPH)
       public CallGraphSnippet readGraph(String[] files,
                                         long     profileId,
                                         int      direction,
                                         int      levels,
                                         int      detail)
       {
          if (levels == -1)
          {
             List<Vertex> lf = db.traversal().V().has("node-type", PROCEDURE_FILE).toList();
             files = new String[lf.size()];
             int i = 0;
             for (Vertex v : lf)
             {
                files[i++] = v.<String>property("os-filename").value() + ".ast";
             }
          }
          else if (profileId > 0)
          {
             List<String> fl = new ArrayList<>();

             // read a file set
             for (String f : ReportApi.listFilesFromProfile(profileId))
             {
                fl.add(f + ".ast");
             }

             files = new TreeSet<String>(fl).toArray(new String[0]);
          }
          else if (files == null)
          {
             try
             {
                // read root node list from XML
                RootNodeList rnl = new RootNodeList();
                files = rnl.getList();
             }

             catch (ConfigurationException | FileNotFoundException e)
             {
                // fall through
                LOG.log(Level.SEVERE, "", e);
             }
          }

          if (files != null)
          {
              for (int i = 0; i < files.length; i++)
              {
                 if (!files[i].endsWith(".ast"))
                 {
                    files[i] += ".ast";
                 }
              }
          }

          Set<Vertex> nodes = new LinkedHashSet<>();
          Set<Edge> links = new LinkedHashSet<>();
          Set<CallGraphLink> virtualLinks = new LinkedHashSet<>();
          Set<CallGraphNode> moreNodes = new LinkedHashSet<>();
          Set<CallGraphLink> moreLinks = new LinkedHashSet<>();

          // the next link ID
          AtomicInteger nextLinkId = new AtomicInteger(0);

          Set<Vertex> start = new LinkedHashSet<>(CallGraphHelper.loadFileNodes(files, true));

          if (start != null)
          {
             // recursively process each level of the graph and store the results
             // in the given sets
             Map<Vertex, List<Edge>> level0 = traverseLevel(start,
                                                            direction,
                                                            levels,
                                                            detail,
                                                            nodes,
                                                            links,
                                                            virtualLinks,
                                                            nextLinkId);

             Map<CallGraphLink, CallGraphLink> p2pLinks = new HashMap<>();

             // level0 targets need to be processed here, as we need to be sure they are real level0
             for (Vertex v : level0.keySet())
             {
                List<Edge> edges = level0.get(v);

                Vertex vParent = computeParentForDetail(v, detail);
                if (vParent == null)
                {
                   // not from my level of detail
                   continue;
                }

                int vresDom = v.<Integer>property("resource-domain").value();
                Vertex vFile = null;
                if (vresDom == AST_NODE || vresDom == EVENT)
                {
                   vFile = CallGraphHelper.findProcedureFile(v);
                   if (vFile == null)
                   {
                      vFile = CallGraphHelper.findParentAstVertex(v, SCHEMA_FILE);
                   }
                }

                // process each edge
                for (Edge e : edges)
                {
                   // don't process links twice
                   if (links.contains(e))
                   {
                      continue;
                   }

                   // find the vertext on the other side of the edge
                   boolean in = e.inVertex().equals(v);
                   Vertex other = in ? e.outVertex() : e.inVertex();
                   Vertex otherParent = computeParentForDetail(other, detail);

                   if (otherParent == null)
                   {
                      // not on the detail level
                      continue;
                   }

                   if (detail == GRAPH_DETAIL_ALL)
                   {
                      // the node is already part of the graph and we don't need a placeholder
                      if (nodes.contains(other))
                      {
                         links.add(e);
                         continue;
                      }
                   }
                   else
                   {
                      if (nodes.contains(otherParent) && nodes.contains(vParent))
                      {
                         virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                            (!in ? vParent : otherParent).id().toString(),
                                                            (in ? vParent : otherParent).id().toString(),
                                                            e.label()));
                         continue;
                      }
                   }

                   // here, we need to determine if the target is a MORE node or not.
                   // a MORE node can be only a PROCEDURE_FILE or a SCHEMA_FILE, and only if the
                   // vertices in this edge are not in the same file

                   // determine the node's file
                   int oresDom = other.<Integer>property("resource-domain").value();
                   Vertex ovFile = null;
                   if (oresDom == AST_NODE || oresDom == EVENT)
                   {
                      ovFile = CallGraphHelper.findProcedureFile(other);
                      if (ovFile == null)
                      {
                         ovFile = CallGraphHelper.findParentAstVertex(other, SCHEMA_FILE);
                      }
                   }

                   if (ovFile != null && vFile != null)
                   {
                      if (ovFile.id().equals(vFile.id()))
                      {
                         if (detail == GRAPH_DETAIL_ALL)
                         {
                            // in the same file, just add them
                            links.add(e);
                            nodes.add(other);
                         }
                         else
                         {
                            nodes.add(otherParent);
                            virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                               (!in ? vParent : otherParent).id().toString(),
                                                               (in ? vParent : otherParent).id().toString(),
                                                               e.label()));
                         }
                      }
                      else
                      {
                         // transform it into a MORE node and create a link to it, from file to file
                         String otherId   = ovFile.id().toString();
                         int    atype = ovFile.<Integer>property("node-type").value();
                         String text  = describeNode(ovFile, atype);
                         String tooltip = describeTooltip(ovFile, atype);

                         moreNodes.add(new CallGraphNode(otherId, MORE, text, tooltip));

                         String srcID = in ? otherId : vFile.id().toString();
                         String tarID = in ? vFile.id().toString() : otherId;

                         CallGraphLink parentLink = new CallGraphLink(nextLinkId.get(),
                                                                      srcID,
                                                                      tarID,
                                                                      e.label());
                         if (p2pLinks.containsKey(parentLink))
                         {
                            parentLink = p2pLinks.get(parentLink);
                         }
                         else
                         {
                            // create a link to it
                            moreLinks.add(parentLink);
                            p2pLinks.put(parentLink, parentLink);
                            nextLinkId.incrementAndGet();
                         }

                         // for this inter-program link, create an explicit link and set its parent
                         // now
                         String vId = (detail == GRAPH_DETAIL_ALL ? v : vParent).id().toString();

                         moreLinks.add(new CallGraphLink(nextLinkId.get(),
                                                         parentLink.getId(),
                                                         in ? otherId : vId,
                                                         in ? vId : otherId,
                                                         e.label()));
                         nextLinkId.incrementAndGet();
                      }
                   }
                   else
                   {
                      // in this case, the other side of the link is something else (i.e. an
                      // include file, an external resource, etc). so, just link to it

                      if (detail == GRAPH_DETAIL_ALL)
                      {
                         nodes.add(other);
                         links.add(e);
                      }
                      else
                      {
                         nodes.add(otherParent);
                         virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                            (!in ? vParent : otherParent).id().toString(),
                                                            (in ? vParent : otherParent).id().toString(),
                                                            e.label()));
                      }
                   }
                }
             }

             moreLinks.addAll(virtualLinks);
          }

          return createSnippet(nextLinkId, nodes, links, moreNodes, moreLinks, files);
       }

       /**
        * List all the call-sites targeting this vertex from the callgraph.
        *
        * @param    vertexId
        *           The vertex ID.
        *
        * @return   Details about all the call-sites incoming in this vertex.
        */
       @WebApi(type = MST_REPORT_CALL_SITES)
       public List<CallSite> getCallSites(String vertexId)
       {
          List<CallSite> res = new ArrayList<>();

          Iterator<Vertex> iter = db.traversal()
                                    .V(Long.parseLong(vertexId))
                                    .inE("calls", "references")
                                    .outV();
          while (iter.hasNext())
          {
             Vertex out = iter.next();

             CallSite cs = buildCallSite(out);

             if (cs != null)
             {
                res.add(cs);
             }
          }

          db.tx().commit();

          return res;
       }

       /**
        * Get the external targets marked as missing.
        *
        * @return   The details for the missing targets.
        */
       @WebApi(type = MSG_REPORT_MISSING_TARGETS)
       public List<CallGraphReportLine> getMissingExternalTargets()
       {
          Iterator<Map<Object, Long>> iter = db.traversal().V()
                                               .has("external", true)
                                               .has("missing", true)
                                               .inE("calls", "references")
                                               .groupCount().by(e -> ((Edge) e).inVertex().id());
          Map<Object, Long> m = iter.hasNext() ? iter.next() : Collections.emptyMap();

          List<CallGraphReportLine> res = new ArrayList<>();

          Iterator<Object> viter = m.keySet().iterator();
          while (viter.hasNext())
          {
             Object vid = viter.next();
             Long count = m.get(vid);
             Vertex v = db.traversal().V(vid).next();

             String type = v.<String>property("node-key").value();
             switch (type)
             {
                case "filename":
                   type = "External Program";
                   break;
                case "missing_function":
                   type = "Function";
                   break;
                case "missing_procedure":
                   type = "Internal Procedure";
                   break;
             }

             CallGraphReportLine mct = new CallGraphReportLine();
             mct.setType(type);
             mct.setName(CallGraphHelper.getNodeKeyValue(v));
             mct.setCallNum(count.intValue());
             mct.setVertexId(v.id().toString());

             res.add(mct);
          }

          Collections.sort(res, new Comparator<CallGraphReportLine>()
          {
             @Override
             public int compare(CallGraphReportLine c1, CallGraphReportLine c2)
             {
                if (c1.getType().equals(c2.getType()))
                {
                   return c1.getName().compareTo(c2.getName());
                }

                return c1.getType().compareTo(c2.getType());
             }
          });

          db.tx().commit();

          return res;
       }

       /**
        * Compute the external dependencies, which are not marked as missing.
        *
        * @return   The report, as a list of external dependencies.
        */
       @WebApi(type = MST_REPORT_EXTERNAL_DEP)
       public List<CallGraphReportLine> getExternalDependencies()
       {
          Iterator<Map<Object, Long>> iter = db.traversal().V()
                                               .has("external", true)
                                               .hasNot("missing")
                                               .inE("calls", "references")
                                               .groupCount().by(e -> ((Edge) e).inVertex().id());
          Map<Object, Long> m = iter.hasNext() ? iter.next() : Collections.emptyMap();

          List<CallGraphReportLine> res = new ArrayList<>();

          Iterator<Object> viter = m.keySet().iterator();
          while (viter.hasNext())
          {
             Object vid = viter.next();
             Long count = m.get(vid);
             Vertex v = db.traversal().V(vid).next();

             int nodeType = v.<Integer>property("node-type").value();

             String nodeKey = v.<String>property("node-key").value();
             String type = v.label();

             if (nodeType == NATIVE_API)
             {
                Vertex vLibRef = v.edges(Direction.IN, "contains").next().outVertex();
                type = vLibRef.label() + " - " +  CallGraphHelper.getNodeKeyValue(vLibRef);
             }

             CallGraphReportLine mct = new CallGraphReportLine();
             mct.setType(type);
             mct.setName(v.<String>property(nodeKey).value());
             mct.setCallNum(count.intValue());
             mct.setVertexId(v.id().toString());

             res.add(mct);
          }

          Collections.sort(res, new Comparator<CallGraphReportLine>()
          {
             @Override
             public int compare(CallGraphReportLine c1, CallGraphReportLine c2)
             {
                if (c1.getType().equals(c2.getType()))
                {
                   return c1.getName().compareTo(c2.getName());
                }

                return c1.getType().compareTo(c2.getType());
             }
          });

          db.tx().commit();

          return res;
       }

       /**
        * For a given vertex ID assumed as dead code, return all the vertex ASTs which are marked
        * as 'contains' or 'includes', directly from this vertex.
        *
        * @param     vertexId
        *            The vertex ID.
        *
        * @return    The report details.
        */
       @WebApi(type = MST_REPORT_DEAD_DEPENDENCIES)
       public List<CallSite> getDeadDependencies(String vertexId)
       {
          List<CallSite> res = new ArrayList<>();

          Iterator<Vertex> iter = db.traversal()
                                    .V(Long.parseLong(vertexId))
                                    .outE("contains", "includes")
                                    .inV()
                                    .hasNot("reachable")
                                    .hasNot("virtual");
          while (iter.hasNext())
          {
             Vertex in = iter.next();

             CallSite callSite = buildCallSite(in);

             if (callSite == null)
             {
                CallSite cs = new CallSite();
                int nodeType = in.<Integer>property("node-type").value();

                cs.setTokenText(ProgressParser.lookupTokenName(nodeType));
                cs.setCallSite(describeNode(in, nodeType));

                if (nodeType == INCLUDE_FILE)
                {
                   in.edges(Direction.IN, "includes").forEachRemaining(e ->
                   {
                      if (e.outVertex().id().toString().equals(vertexId) && e.inVertex().equals(in))
                      {
                         int column = e.<Integer>property("column").value();
                         int line = e.<Integer>property("line").value();

                         cs.setColumn(column);
                         cs.setLine(line);
                      }
                   });
                }

                callSite = cs;
             }

             res.add(callSite);
          }

          Collections.sort(res, new Comparator<CallSite>()
          {
             @Override
             public int compare(CallSite o1, CallSite o2)
             {
                return o1.getTokenText().equals(o2.getTokenText())
                         ? o1.getCallSite().compareTo(o2.getCallSite())
                         : o1.getTokenText().compareTo(o2.getTokenText());
             }

          });

          db.tx().commit();

          return res;
       }

       /**
        * Report all dead code entry-points: programs, internal procedures/functions, trigger blocks,
        * OO classes, properties and methods.
        *
        * @return   The report details.
        */
       @WebApi(type = MST_REPORT_DEAD_CODE)
       public List<DeadCodeReportLine> getDeadCode()
       {
          long start = System.currentTimeMillis();
          List<DeadCodeReportLine> res = new ArrayList<>();
          Set<Object> reached = new HashSet<>();
          String parentProp = CallGraphHelper.parentProperty(PROCEDURE_FILE);

          Consumer<Vertex> collect = vk ->
          {
             if (vk.property("virtual").isPresent())
             {
                return;
             }

             VertexProperty<Integer> vp = vk.<Integer>property(parentProp);
             Vertex v = vk;
             if (vp.isPresent())
             {
                v = db.traversal().V(vp.value()).next();
             }

             if (reached.contains(v))
             {
                return;
             }

             reached.add(v);

             VertexProperty<String> vpFile = v.<String>property("filename");
             String pfile = vpFile.isPresent() ? vpFile.value() : CallGraphHelper.getNodeKeyValue(v);
             int resDom = v.<Integer>property("resource-domain").value();
             int nodeType = v.<Integer>property("node-type").value();

             DeadCodeReportLine dc = new DeadCodeReportLine();
             dc.setFile(pfile);
             dc.setType(ProgressParser.lookupTokenName(resDom == AST_NODE ? PROCEDURE_FILE : nodeType));
             dc.setVertexId(v.id().toString());

             res.add(dc);
          };

          // get all nodes with no "reachable" property, of the specified types
          db.traversal().V()
            .has("node-type",
                 within(FUNCTION,
                        INTERNAL_PROCEDURE,
                        METHOD_DEF,
                        DEFINE_PROPERTY_SET,
                        DEFINE_PROPERTY_GET,
                        CONSTRUCTOR,
                        DESTRUCTOR,
                        CLASS_DEF,
                        INTERFACE_DEF,
                        ENUM_DEF,
                        EXTERNAL_PROCEDURE,
                        PROCEDURE_FILE))
            .has("resource-domain", AST_NODE)
            .has("reachable", neq(true))
            .forEachRemaining(v -> collect.accept(v));

          // get all nodes with no "reachable" property, of the specified types
          db.traversal().V()
            .has("node-type", INCLUDE_FILE)
            .has("resource-domain", FILE_RESOURCE)
            .has("reachable", neq(true))
            .forEachRemaining(v -> collect.accept(v));

          // get all nodes with no "reachable" property, of the specified types
          db.traversal().V()
            .has("external", true)
            .has("missing", neq(true))
            .has("reachable", neq(true))
            .forEachRemaining(v -> collect.accept(v));

          Collections.sort(res, new Comparator<DeadCodeReportLine>()
          {
             @Override
             public int compare(DeadCodeReportLine c1, DeadCodeReportLine c2)
             {
                if (c1.getType().equals(c2.getType()))
                {
                   return c1.getFile().compareTo(c2.getFile());
                }
                else
                {
                   return c1.getType().compareTo(c2.getType());
                }
             }
          });

          long end = System.currentTimeMillis();
          System.out.println(String.format("Dead code done in %d millis", end - start));

          db.tx().commit();

          return res;
       }

       /**
        * Get all the program files with ambiguous call-sites.
        *
        * @return   The list of program files with ambiguous call-sites.
        */
       @WebApi(type = MST_REPORT_AMBIG_CALLSITES)
       public List<AmbigFileReportLine> getAmbiguousFiles()
       {
          Vertex vAmbig = CallGraphHelper.findNodeById(AMBIGUOUS, CallGraphHelper.AMBIGUOUS_NODE_ID);

          Iterator<Map<Object, Long>> iter = db.traversal().V(vAmbig).inE().outV()
                                               .groupCount()
                                               .by(CallGraphHelper.parentProperty(PROCEDURE_FILE));
          Map<Object, Long> groups = iter.hasNext() ? iter.next() : Collections.emptyMap();
          Iterator<Object> iter2 = groups.keySet().iterator();

          Map<String, Integer> refs = new HashMap<>();
          while (iter2.hasNext())
          {
             Long fileId = (Long) iter2.next();

             if (!db.traversal().V(fileId).has("finished", true).hasNext())
             {
                // show only files which have been processed by the call graph
                continue;
             }

             String filename = (String) db.traversal().V(fileId).properties("filename").value().next();
             if (!refs.containsKey(filename))
             {
                refs.put(filename, 0);
             }

             Long count = groups.get(fileId);
             refs.put(filename, refs.get(filename) + count.intValue());
          }

          List<AmbigFileReportLine> res = new ArrayList<>();
          for (String filename : new TreeSet<String>(refs.keySet()))
          {
             res.add(new AmbigFileReportLine(filename, refs.get(filename)));
          }

          db.tx().commit();

          return res;
       }

       /**
        * Get all the ambiguous call-sites in the specified program file.
        *
        * @param    filename
        *           The program file name.
        *
        * @return   The list of ambiguous call-sites.
        */
       @WebApi(type = MST_REPORT_FILE_AMBIG_CALLSITES)
       public List<CallSite> getFileAmbiguousCallSites(String filename)
       {
          Vertex vAmbig = CallGraphHelper.findNodeById(AMBIGUOUS, CallGraphHelper.AMBIGUOUS_NODE_ID);
          List<CallSite> res = new ArrayList<>();

          Vertex vfile = db.traversal().V().has("filename", filename).next();

          String procFileProp = CallGraphHelper.parentProperty(PROCEDURE_FILE);
          Iterator<Vertex> iter = db.traversal().V(vAmbig).inE().outV().has(procFileProp, vfile.id());

          while (iter.hasNext())
          {
             Vertex v = iter.next();

             CallSite cs = buildCallSite(v);

             if (cs != null)
             {
                Iterator<Edge> eiter = v.edges(Direction.OUT, "ambiguous");
                if (eiter.hasNext())
                {
                   int hintId = (Integer) CallGraphHelper.property(v, "hint-instance");
                   String tokType = ProgressParser.lookupTokenName(
                                       v.<Integer>property("node-type").value());
                   String hint = String.format("%s_%s", tokType, hintId);
                   cs.setHint(hint);
                   cs.setFilename(filename + ".hints");
                   res.add(cs);

                   Integer includeHintId =
                              (Integer) CallGraphHelper.property(v, "include-hint-instance");
                   if (includeHintId != null)
                   {
                      String includeHintFile = (String) CallGraphHelper.property(v, "include-hint-file");
                      String includeHint = String.format("%s_%s", tokType, includeHintId);

                      cs = new CallSite(cs);
                      cs.setFilename(includeHintFile + ".hints");
                      cs.setHint(includeHint);
                      res.add(cs);
                   }
                }
                else
                {
                   System.out.println("Warning: ambiguous node linked with an incorrect edge");
                   v.properties()
                    .forEachRemaining(vp -> System.out.println(vp.key() + " " + vp.value()));
                   v.edges(Direction.BOTH)
                    .forEachRemaining(e ->
                    {
                       System.out.println(e);
                       e.properties()
                        .forEachRemaining(vp -> System.out.println(vp.key() + " " + vp.value()));
                    });
                }
             }
          }

          Collections.sort(res, new Comparator<CallSite>()
          {
             @Override
             public int compare(CallSite c1, CallSite c2)
             {
                if (c1.getFilename().equals(c2.getFilename()))
                {
                   return c1.getLine() == c2.getLine() ? c1.getColumn() - c2.getColumn()
                                                       : c1.getLine() - c2.getLine();
                }
                else
                {
                   return c1.getFilename().compareTo(c2.getFilename());
                }
             }
          });

          db.tx().commit();

          return res;
       }

       /**
        * Verify the schema triggers loaded in the call graph.  This includes the following problems:
        * <ol>
        *    <li>if a schema trigger links to an external program with no TRIGGER_PROCEDURE</li>
        *    <li>if trigger types don't match</li>
        *    <li>if the schema name differs</li>
        *    <li>if it links to a missing external program</li>
        *    <li>if external procedure with a TRIGGER_PROCEDURE is not linked to any table</li>
        * </ol>
        *
        * @return   The report results.
        */
       @WebApi(type = MST_REPORT_VERIFY_SHEMA_TRIGGERS)
       public List<CallGraphReportLine> getReportVerifySchemaTriggers()
       {
          List<CallGraphReportLine> res = new ArrayList<>();

          // go through all links from a TABLE trigger and report if:
          // 1. it links to an external program with no TRIGGER_PROCEDURE specification
          // 2. if the trigger type matches
          // 3. if the schemaname differs
          // 4. if it links to a missing external program

          // go through all TRIGGER_PROCEDURE vertices and report if:
          // 45. this trigger program is not referenced by any TABLE triggers

          String type1 = "External program is not a table trigger";
          String type2 = "Trigger type mismatch";
          String type3 = "Different table";
          String type4 = "Missing external program";
          String type5 = "Dead table trigger";

          Iterator<Edge> iterE = db.traversal().V().has("node-type", KW_TAB_TRG).outE("calls");
          while (iterE.hasNext())
          {
             Edge e = iterE.next();

             Vertex vTabTrig = e.outVertex();

             Vertex vTable = vTabTrig.vertices(Direction.IN, "contains").next();
             Vertex vExtProg = e.inVertex();

             if (vExtProg.property("missing").isPresent())
             {
                CallGraphReportLine line = new CallGraphReportLine();
                line.setVertexId(vExtProg.id().toString());
                line.setType(type4);
                line.setName(vExtProg.<String>property("filename").value());
                res.add(line);
                continue;
             }

             Vertex vTrigProc = CallGraphHelper.findAdjacent(vExtProg,
                                                             TRIGGER_PROCEDURE,
                                                             "contains",
                                                             true);
             if (vTrigProc == null)
             {
                CallGraphReportLine line = new CallGraphReportLine();
                line.setVertexId(vExtProg.id().toString());
                line.setType(type1);
                line.setName(vExtProg.<String>property("external-procedure").value());
                res.add(line);
                continue;
             }

             int tableTrigType = e.<Integer>property("trigger-type").value();
             int progTrigType = vTrigProc.<Integer>property("trigger-type").value();

             if (tableTrigType != progTrigType)
             {
                CallGraphReportLine line = new CallGraphReportLine();
                line.setVertexId(vExtProg.id().toString());
                line.setType(type2);
                line.setName(vExtProg.<String>property("external-procedure").value());
                res.add(line);
             }

             String tableSchema = vTable.<String>property("schemaname").value();
             String progTableSchema = vTrigProc.<String>property("trigger-schema").value();
             if (!tableSchema.equals(progTableSchema))
             {
                CallGraphReportLine line = new CallGraphReportLine();
                line.setVertexId(vExtProg.id().toString());
                line.setType(type3);
                line.setName(vExtProg.<String>property("external-procedure").value());
                res.add(line);
             }
          }

          Iterator<Vertex> iterV = db.traversal().V().has("node-type", TRIGGER_PROCEDURE);
          while (iterV.hasNext())
          {
             Vertex vTrigProc = iterV.next();

             Vertex vExtProg = vTrigProc.edges(Direction.IN, "contains").next().outVertex();

             boolean hasTableTrigger = false;
             Iterator<Edge> iterT = vExtProg.edges(Direction.IN, "calls");
             while (iterT.hasNext())
             {
                Edge e = iterT.next();
                Vertex vOut = e.outVertex();
                if (vOut.<Integer>property("node-type").value() == KW_TAB_TRG)
                {
                   hasTableTrigger = true;
                   break;
                }
             }

             if (!hasTableTrigger)
             {
                CallGraphReportLine line = new CallGraphReportLine();
                line.setVertexId(vExtProg.id().toString());
                line.setType(type5);
                line.setName(vExtProg.<String>property("external-procedure").value());
                res.add(line);
             }
          }

          Collections.sort(res, new Comparator<CallGraphReportLine>()
          {
             @Override
             public int compare(CallGraphReportLine l1, CallGraphReportLine l2)
             {
                return l1.getType().equals(l2.getType())
                          ? l1.getName().compareTo(l2.getName())
                          : l1.getType().compareTo(l2.getType());
             }
          });

          db.tx().commit();

          return res;
       }

       /**
        * List all the dependencies for the entire program file list.
        * <p>
        * Left side will include the entire loaded code set (which are
        * {@link ProgressParserTokenTypes#FILE_RESOURCE file-level entry points}) and right-side will
        * include the targets to other (non-internal) nodes.
        *
        * @return   The report results.
        */
       @WebApi(type = MST_REPORT_LIST_DEPENDENCIES)
       public List<CallGraphReportLine> getDependenciesReport()
       {
          List<CallGraphReportLine> res = new ArrayList<>();

          db.traversal().V()
            .has("resource-domain", FILE_RESOURCE)
            .has("os-filename")
            .forEachRemaining(v ->
          {
             // avoid this code, as the total time of the report increases
             // int outDependencies = getDependenciesFor(v.id().toString(), true).size();

             // if (outDependencies == 0)
             // {
             //   continue;
             // }

             CallGraphReportLine line = new CallGraphReportLine();
             line.setVertexId(v.id().toString());
             // line.setCallNum(outDependencies);
             line.setName(v.<String>property("os-filename").value());
             line.setType(v.label());

             res.add(line);
          });

          Collections.sort(res, new Comparator<CallGraphReportLine>()
          {
             @Override
             public int compare(CallGraphReportLine l1, CallGraphReportLine l2)
             {
                return l1.getType().equals(l2.getType())
                          ? l1.getName().compareTo(l2.getName())
                          : l1.getType().compareTo(l2.getType());
             }
          });

          db.tx().commit();

          return res;
       }

       /**
        * Get the dependencies for the specified file resources (as vertex ID).
        *
        * @param    vertexId
        *           The vertex ID.
        *
        * @return   The details about the remote call-sites.
        */
       @WebApi(type = MST_REPORT_DEPENDENCIES_FOR)
       public List<CallSite> getDependenciesFor(String vertexId)
       {
          List<CallSite> res = getDependenciesFor(vertexId, false);

          db.tx().commit();

          return res;
       }

       /**
        * Get the dependencies for the specified file resources (as vertex ID).
        *
        * @param    vertexId
        *           The vertex ID.
        * @param    internal
        *           When this flag is set, only first found dependency is reported.
        *
        * @return   The details about the remote call-sites.
        */
       private List<CallSite> getDependenciesFor(String vertexId, boolean internal)
       {
          List<CallSite> res = new ArrayList<>();
          Set<Vertex> nodes = new LinkedHashSet<>();
          Set<Edge> links = new LinkedHashSet<>();

          Vertex v = db.traversal().V(vertexId).next();

          int nodeType = v.<Integer>property("node-type").value();
          boolean isInclude = nodeType == INCLUDE_FILE || v.edges(Direction.IN, "includes").hasNext();

          if (isInclude)
          {
             // include files report only which program files reference them...
             Iterator<Edge> iterE = v.edges(Direction.IN, "includes");

             while (iterE.hasNext())
             {
                Edge e = iterE.next();
                Vertex vOut = e.outVertex();

                Vertex vFile = null;

                if (e.property("call-site-id").isPresent())
                {
                   long parentId = e.<Long>property("call-site-id").value();
                   vFile = CallGraphHelper.findNodeById(PROCEDURE_FILE, parentId);
                }
                else if (vOut.property("filename").isPresent())
                {
                   vFile = vOut;
                }
                else
                {
                   continue;
                }

                CallSite csOut = new CallSite();
                csOut.setFilename("included by " + CallGraphHelper.getNodeKeyValue(vFile));
                csOut.setCallSite(CallGraphHelper.getNodeKeyValue(vOut));

                if (e.property("start-line").isPresent())
                {
                   csOut.setLine(e.<Integer>property("start-line").value());
                   csOut.setColumn(e.<Integer>property("start-column").value());
                }
                else if (vOut.property("line").isPresent())
                {
                   csOut.setLine(vOut.<Integer>property("line").value());
                   csOut.setColumn(vOut.<Integer>property("column").value());
                }
                res.add(csOut);

                if (internal)
                {
                   return res;
                }
             }
          }
          else
          {
             Vertex vp = computeParentForDetail(v, GRAPH_DETAIL_FILES);
             traverseFileResource(v, nodes, links);
             for (Vertex nOut : nodes)
             {
                Iterator<Edge> iterE = nOut.edges(Direction.OUT);

                while (iterE.hasNext())
                {
                   Edge e = iterE.next();

                   // skip the INCLUDES edges
                   if (e.label().equals("includes"))
                   {
                      continue;
                   }

                   Vertex nIn = e.inVertex();

                   // find its parent...
                   Vertex nInParent = computeParentForDetail(nIn, GRAPH_DETAIL_FILES);
                   Vertex nOutParent = computeParentForDetail(nOut, GRAPH_DETAIL_FILES);
                   if (nOutParent == null || !nOutParent.equals(vp))
                   {
                      continue;
                   }

                   if (nInParent != null && !vp.equals(nInParent))
                   {
                      CallSite csOut = buildCallSite(nOut);
                      if (csOut == null)
                      {
                         continue;
                      }

                      String descr = null;
                      if (nInParent.property("os-filename").isPresent())
                      {
                         descr = nInParent.<String>property("os-filename").value();
                      }
                      else
                      {
                         descr = CallGraphHelper.getNodeKeyValue(nInParent);
                      }

                      String nodeKey = nIn.<String>property("node-key").value();
                      if (!nIn.equals(nInParent) &&
                          !(nodeKey.equals("filename")    ||
                            nodeKey.equals("os-filename") ||
                            nodeKey.equals("external-procedure")))
                      {
                         descr = descr + " " + nIn.<String>property(nodeKey).value();
                      }

                      descr = e.label() + " " + nIn.label() + " (" + descr + ")";

                      csOut.setFilename(descr);

                      if (e.property("line").isPresent())
                      {
                         csOut.setLine(e.<Integer>property("line").value());
                      }
                      if (e.property("column").isPresent())
                      {
                         csOut.setColumn(e.<Integer>property("column").value());
                      }

                      res.add(csOut);

                      if (internal)
                      {
                         return res;
                      }
                   }
                }
             }

             // do a DFS search and get the include tree; start with the vertices targeted by
             // INCLUDES edges from the vp node.

             Queue<Edge> inext = new ArrayDeque<>();
             vp.edges(Direction.OUT, "includes").forEachRemaining(edge -> inext.add(edge));

             while (!inext.isEmpty())
             {
                Edge e = inext.poll();

                Vertex vOut = e.outVertex();
                Vertex vIn = e.inVertex();

                CallSite csOut = new CallSite();
                csOut.setFilename("includes " + CallGraphHelper.getNodeKeyValue(vIn));
                csOut.setCallSite(CallGraphHelper.getNodeKeyValue(vOut));

                if (e.property("start-line").isPresent())
                {
                   csOut.setLine(e.<Integer>property("start-line").value());
                   csOut.setColumn(e.<Integer>property("start-column").value());
                }
                else if (vOut.property("line").isPresent())
                {
                   csOut.setLine(vOut.<Integer>property("line").value());
                   csOut.setColumn(vOut.<Integer>property("column").value());
                }
                res.add(csOut);

                if (internal)
                {
                   return res;
                }

                vIn.edges(Direction.OUT, "includes").forEachRemaining(edge ->
                {
                   Property<String> p = edge.<String>property("parent-edge-id");
                   if (p.isPresent() && p.value().equals(e.id().toString()))
                   {
                      inext.add(edge);
                   }
                });
             }
          }

          Collections.sort(res, new Comparator<CallSite>()
          {
             @Override
             public int compare(CallSite c1, CallSite c2)
             {
                if (c1.getFilename().equals(c2.getFilename()))
                {
                   return c1.getLine() == c2.getLine() ? c1.getColumn() - c2.getColumn()
                                                       : c1.getLine() - c2.getLine();
                }
                else
                {
                   return c1.getFilename().compareTo(c2.getFilename());
                }
             }
          });

          return res;
       }

       /**
        * Traverse the file resource and load the entire "contains" or "includes" tree.
        *
        * @param    start
        *           The node to start with.
        * @param    nodes
        *           The vertex collector.
        * @param    links
        *           The link collector.
        */
       private void traverseFileResource(Vertex start, Set<Vertex> nodes, Set<Edge> links)
       {
          if (nodes.contains(start))
          {
             return;
          }

          nodes.add(start);

          String[] lbl = start.<Integer>property("node-type").value() == INCLUDE_FILE
                           ? new String[] { "includes" }
                           : new String[] { "contains", "raises", "listens", "registers" };

          Iterator<Edge> iter = db.traversal().V(start).outE(lbl);

          while (iter.hasNext())
          {
             Edge nextE = iter.next();
             Vertex nextV = nextE.inVertex();

             links.add(nextE);

             traverseFileResource(nextV, nodes, links);
          }
       }

       /**
        * Gather the vertices and edges for the specified portion of the graph and optionally
        * use recursion to get each additional requested level until all levels are gathered.
        *
        * @param    start
        *           The list of vertices to use as the starting point.
        * @param    direction
        *           One of the direction constants from this class for {@link #BOTH_DIRECTIONS},
        *           {@link #DOWNSTREAM} or {@link #UPSTREAM}. This controls which edges (inbound
        *           and/or outbound) are included.  UPSTREAM maps to inbound and DOWNSTREAM maps
        *           to outbound.
        * @param    levels
        *           This defines the number of cross-file levels of traversal to include in the
        *           snippet. Since some nodes are contained (using edges labeled "contains") in
        *           other nodes, using a simple calculation of edge levels would lead to inconsistent
        *           results.  Instead, we track the number of times that the graph traveral crosses
        *           from one file domain resource into another file domain resource. Use 0 to
        *           include only the starting nodes, 1 to include the starting nodes plus the nodes
        *           that can be traversed through 1 edge (in the direction specified), 2 for
        *           traversal through 2 edges and so forth. -1 means there is no limit. Use the no
        *           limit options with caution as there may be performance and memory issues with
        *           the result.
        * @param    detail
        *           The level of detail for the graph.
        * @param    nodes
        *           The set in which to gather the vertices.
        * @param    links
        *           The set in which to gather the edges.
        * @param    virtualLinks
        *           For a graph detail other than {@link #GRAPH_DETAIL_ALL}, it will contain virtual
        *           links between the vertices which are not expanded.
        * @param    nextLinkId
        *           The counter which generates link IDs.
        *
        * @return   The level-0 assumed edges, per each vertex.  These edges will be processed against
        *           the final graph, to determine if they are real level-0 or not.
        */
       private Map<Vertex, List<Edge>> traverseLevel(Set<Vertex>        start,
                                                     int                direction,
                                                     int                levels,
                                                     int                detail,
                                                     Set<Vertex>        nodes,
                                                     Set<Edge>          links,
                                                     Set<CallGraphLink> virtualLinks,
                                                     AtomicInteger      nextLinkId)
       {
          if (start.isEmpty())
          {
             return Collections.emptyMap();
          }

          Map<Vertex, List<Edge>> level0 = new LinkedHashMap<>();

          // make sure that these nodes are added to the graph
          nodes.addAll(start);

          String[] edgeLbls = { "calls", "raises", "listens", "registers", "references" };

          Set<Vertex> newNodes = new LinkedHashSet<>();

          // for each vertex in the starting list
          for (Vertex s : start)
          {
             Set<Vertex> contained = new LinkedHashSet<Vertex>();
             Set<Edge> containedLinks = new LinkedHashSet<Edge>();

             traverseFileResource(s, contained, containedLinks);

             // the above will produce the entire sub-graph for this file; from these, keep only nodes
             // which are involved in the callgraph ('calls', 'listens' and 'raises'), incoming or
             // outgoing

             Set<Vertex> keepNodes = new HashSet<>();
             Set<Edge> keepLinks = new HashSet<>();
             for (Vertex v : contained)
             {

                VertexProperty<Boolean> vEntryPoint = v.property("entry-point");

                if (vEntryPoint.isPresent() && vEntryPoint.value() && !v.property("virtual").isPresent())
                {
                   // always keep all defined entry points
                   keepNodes.add(v);
                   v.edges(Direction.IN, "contains").forEachRemaining(e -> keepLinks.add(e));
                }

                Iterator<Edge> iter = v.edges(Direction.BOTH, edgeLbls);

                if (iter.hasNext())
                {
                   Vertex vp = v;

                   while (vp != null)
                   {
                      keepNodes.add(vp);

                      Iterator<Edge> iter2 = db.traversal().V(vp).inE("contains");
                      if (iter2.hasNext())
                      {
                         Edge ep = iter2.next();
                         keepLinks.add(ep);

                         vp = ep.outVertex();
                      }
                      else
                      {
                         vp = null;
                      }
                   }

                   // these are all in the current context
                   String[] lbls = { "raises", "listens", "registers" };
                   for (String lbl : lbls)
                   {
                      Iterator<Edge> iter2 = db.traversal().V(v).bothE(lbl);
                      if (iter2.hasNext())
                      {
                         Edge ep = iter2.next();
                         keepLinks.add(ep);
                         keepNodes.add(ep.outVertex());
                         keepNodes.add(ep.inVertex());
                      }
                   }
                }
             }

             {
                // the 'contained' sets must remain with the 'keep' sets, but preserving order... so
                // this difference quirk is used
                Set<Vertex> s1 = new HashSet<>(contained);
                s1.removeAll(keepNodes);
                contained.removeAll(s1);

                Set<Edge> s2 = new HashSet<>(containedLinks);
                s2.removeAll(keepLinks);
                containedLinks.removeAll(s2);
             }

             // save the links...
             if (detail == GRAPH_DETAIL_ALL)
             {
                links.addAll(containedLinks);
             }
             else if (detail == GRAPH_DETAIL_ENTRY_POINTS)
             {
                for (Edge e : containedLinks)
                {
                   Vertex vIn = e.inVertex();
                   Vertex vOut = e.outVertex();

                   Vertex vTargetIn = computeParentForDetail(vIn, detail);
                   Vertex vTargetOut = computeParentForDetail(vOut, detail);
                   // for this case, keep only the intra-links which reach the detail level (i.e.
                   // 'contains' links from program file to function, internal proc, etc
                   if (vIn.equals(vTargetIn) && vOut.equals(vTargetOut))
                   {
                      newNodes.add(vTargetOut);
                      newNodes.add(vTargetIn);

                      // the link is a 'contains' intra-link with the proper detail level, keep it
                      virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                         vTargetOut.id().toString(),
                                                         vTargetIn.id().toString(),
                                                         e.label()));
                   }
                }
             }
             else
             {
                // for the GRAPH_DETAIL_FILES, the 'contained' links are not inter-files links, so
                // do not add anything
             }

             // process all of the contained nodes for the current node in our starting list
             for (Vertex v : contained)
             {
                // don't process nodes twice
                if (nodes.contains(v))
                {
                   continue;
                }

                Vertex parent = computeParentForDetail(v, detail);
                if (parent != null)
                {
                   // save the parent node (as this is the detail level)
                   nodes.add(parent);
                }
                else
                {
                   // parent can't be found, so what to do?
                   // TODO: !!!!!!!!!
                   continue;
                }

                List<Edge>   edges = null;
                List<Vertex> next  = null;

                // gather the non-contains edges from those contained nodes
                if (direction == DOWNSTREAM)
                {
                   edges = db.traversal().V(v).outE(edgeLbls).toList();
                   next  = db.traversal().V(v).out(edgeLbls).toList();
                }
                else if (direction == UPSTREAM)
                {
                   edges = db.traversal().V(v).inE(edgeLbls).toList();
                   next  = db.traversal().V(v).in(edgeLbls).toList();
                }
                else
                {
                   // otherwise assume BOTH_DIRECTIONS
                   edges = db.traversal().V(v).bothE(edgeLbls).toList();
                   next  = db.traversal().V(v).both(edgeLbls).toList();
                }

                // edges might point to external resources which are not single-node (i.e. native
                // API calls, missing external procedures/internal procedures/functions).  These
                // cases can't be treated as MORE nodes (as their target is an actual program file)
                // so they need to be expanded fully here
                Iterator<Edge> iterE = edges.iterator();
                while (iterE.hasNext())
                {
                   Edge e = iterE.next();
                   Vertex vOther = (v.equals(e.inVertex()) ? e.outVertex() : e.inVertex());

                   if (vOther.property("external").isPresent() &&
                       vOther.<Boolean>property("external").value())
                   {
                      next.remove(vOther);
                      iterE.remove();

                      Iterator<Edge> iterEContains = vOther.edges(Direction.IN, "contains");
                      if (iterEContains.hasNext())
                      {
                         // there is only one level of contains here. get the parent and add all its
                         // children, to have a complete state
                         Edge eContains = iterEContains.next();
                         Vertex vOtherParent = eContains.outVertex();

                         nodes.add(vOtherParent);

                         if (detail == GRAPH_DETAIL_FILES)
                         {
                            // create a 'to-file' link and hide its internal representation
                            Vertex vParent = CallGraphHelper.findProcedureFile(v);
                            if (vParent == null)
                            {
                               vParent = CallGraphHelper.findSchemaFile(v);
                            }

                            nodes.add(vParent);
                            virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                               (v.equals(e.outVertex()) ? vParent.id() : vOtherParent.id()).toString(),
                                                               (v.equals(e.inVertex()) ? vParent.id() : vOtherParent.id()).toString(),
                                                               e.label()));
                         }
                         else
                         {
                            // just add the link, is not hidden
                            links.add(e);

                            iterEContains = vOtherParent.edges(Direction.OUT, "contains");
                            while (iterEContains.hasNext())
                            {
                               eContains = iterEContains.next();
                               links.add(eContains);
                               nodes.add(eContains.inVertex());
                            }
                         }
                      }
                      else
                      {
                         Vertex vOut = e.outVertex();
                         Vertex vIn = e.inVertex();

                         Vertex vTargetOut = computeParentForDetail(vOut, detail);
                         Vertex vTargetIn = computeParentForDetail(vIn, detail);

                         if (vTargetOut != null && vTargetIn != null && !vTargetOut.equals(vTargetIn))
                         {
                            virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                               vTargetOut.id().toString(),
                                                               vTargetIn.id().toString(),
                                                               e.label()));
                            nodes.add(vTargetOut);
                            nodes.add(vTargetIn);
                         }
                      }
                   }
                }

                if (levels == 0)
                {
                   // on the depth-first search being done here, although an edge might look like it
                   // points outside of our max-level structure, another parallel path might meet the
                   // same 'remote' vertex sooner, so this node can't become a MORE node; for this
                   // reason, we collect 'level0' edges and process them only after the entire DFS
                   // has finished
                   if (!edges.isEmpty())
                   {
                      level0.put(v, edges);
                   }
                }
                else
                {
                   Set<Vertex> nextStart = new LinkedHashSet<>();
                   for (Vertex n : next)
                   {
                      int nodeType = n.<Integer>property("node-type").value();
                      int resDom = n.<Integer>property("resource-domain").value();

                      if (resDom == AST_NODE || resDom == EVENT)
                      {
                         Vertex pNode = CallGraphHelper.findProcedureFile(n);

                         if (pNode != null)
                         {
                            nextStart.add(pNode);
                         }
                         else
                         {
                            pNode = CallGraphHelper.findSchemaFile(n);
                            if (pNode != null)
                            {
                               nextStart.add(pNode);
                            }
                         }
                      }
    /*
                      else if (nodeType == INCLUDE_FILE)
                      {
                         nextStart.add(n);
                      }
    */
                      else if (n.property("external").isPresent() &&
                               n.<Boolean>property("external").value())
                      {
                         Vertex pNode = n;

                         if (nodeType == NATIVE_API         ||
                             nodeType == INTERNAL_PROCEDURE ||
                             nodeType == FUNCTION)
                         {
                            // find the parent container for a native API (shared library) and for
                            // missing internal proc/function (the external program, missing or not)
                            Iterator<Vertex> iter = n.vertices(Direction.IN, "contains");

                            pNode = (iter.hasNext() ? iter.next() : null);
                         }

                         if (pNode != null)
                         {
                            nextStart.add(pNode);
                         }
                      }
                   }

                   // remove the already walked nodes...
                   nextStart.removeAll(nodes);

                   // store the edges in our results
                   if (detail == GRAPH_DETAIL_ALL)
                   {
                      links.addAll(edges);
                   }
                   else
                   {
                      for (Edge e : edges)
                      {
                         Vertex vOut = e.outVertex();
                         Vertex vIn = e.inVertex();

                         Vertex vTargetOut = computeParentForDetail(vOut, detail);
                         Vertex vTargetIn = computeParentForDetail(vIn, detail);

                         if (vTargetOut != null && vTargetIn != null && !vTargetOut.equals(vTargetIn))
                         {
                            // nodes.add(vTargetOut);
                            // nodes.add(vTargetIn);

                            virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                                               vTargetOut.id().toString(),
                                                               vTargetIn.id().toString(),
                                                               e.label()));
                         }
                      }
                   }
                   nodes.addAll(newNodes);
                   newNodes.clear();

                   // there is another level to gather, recurse, reducing the level parm value
                   level0.putAll(traverseLevel(nextStart,
                                               direction,
                                               levels - 1,
                                               detail,
                                               nodes,
                                               links,
                                               virtualLinks,
                                               nextLinkId));
                }
             }
          }

          return level0;
       }

       /**
        * Depending on the graph detail level, it will compute the parent vertex to be added to the
        * emitted graph snippet.
        *
        * @param    v
        *           The vertex.
        * @param    detail
        *           The graph detail level.
        *
        * @return   If the detail is {@link #GRAPH_DETAIL_ALL}, return the vertex.  Otherwise, one of
        *           its parents or {@code null} if the parent or vertex type doesn't match the detail
        *           level.
        */
       private Vertex computeParentForDetail(Vertex v, int detail)
       {
          Vertex parent = null;
          int resDomain = v.<Integer>property("resource-domain").value();
          int nodeType = v.<Integer>property("node-type").value();
          boolean external = v.<Boolean>property("external").isPresent() &&
                             v.<Boolean>property("external").value();

          switch (detail)
          {
             case GRAPH_DETAIL_FILES:
                if (resDomain == FILE_RESOURCE)
                {
                   return v;
                }

                if (resDomain == AST_NODE)
                {
                   parent = CallGraphHelper.findProcedureFile(v);

                   if (parent == null)
                   {
                      parent = CallGraphHelper.findSchemaFile(v);
                   }

                   return parent;
                }

                // check if this is a library procedure
                if (nodeType == NATIVE_API)
                {
                   return v.edges(Direction.IN, "contains").next().outVertex();
                }

                // what's left is an external target
                return external || nodeType == INCLUDE_FILE ? v : null;

             case GRAPH_DETAIL_ENTRY_POINTS:
                if (resDomain == FILE_RESOURCE || nodeType == NATIVE_API)
                {
                   return v;
                }

                if (resDomain == AST_NODE)
                {
                   int[] parents =
                   {
                      FUNCTION,
                      INTERNAL_PROCEDURE,
                      METHOD_DEF,
                      DEFINE_PROPERTY_SET,
                      DEFINE_PROPERTY_GET,
                      CONSTRUCTOR,
                      DESTRUCTOR,
                      CLASS_DEF,
                      INTERFACE_DEF,
                      ENUM_DEF,
                      EXTERNAL_PROCEDURE,
                      PROCEDURE_FILE,
                      TABLE,
                      DATABASE,
                      SCHEMA_FILE
                   };

                   for (int ptype : parents)
                   {
                      parent = CallGraphHelper.findParentAstVertex(v, ptype);
                      if (parent != null)
                      {
                         break;
                      }
                   }

                   return parent;
                }

                return external ? v : null;

             case GRAPH_DETAIL_ALL:
                return v;
          }

          throw new IllegalArgumentException("Invalid graph detail level: " + detail);
       }

       /**
        * Render the given sets of nodes and links as a call graph snippet. Since this is
        * the last thing done with the graph database for the given API request, we commit
        * the current transaction at the end of the rendering work.
        *
        * @param    nextLinkId
        *           The counter which generates link IDs.
        * @param    nodes
        *           The vertices to include.
        * @param    links
        *           The edges to include.
        * @param    moreNodes
        *           The set in which to gather the artificial placeholder vertices.
        * @param    moreLinks
        *           The set in which to gather the artificial placeholder edges.
        * @param    rootList
        *           The list of root program files used to build the graph snippet.
        *
        * @return   The snippet.
        */
       private CallGraphSnippet createSnippet(AtomicInteger      nextLinkId,
                                              Set<Vertex>        nodes,
                                              Set<Edge>          links,
                                              Set<CallGraphNode> moreNodes,
                                              Set<CallGraphLink> moreLinks,
                                              String[]           rootList)
       {
          List<CallGraphNode>   nodeList   = new ArrayList<>();
          List<CallGraphLink>   linkList   = new ArrayList<>();
          Map<String, CallSite> astDetails = new HashMap<>();
          Set<String> nodeIds = new HashSet<>();

          for (Vertex v : nodes)
          {
             nodeIds.add(v.id().toString());
          }

          for (CallGraphNode n : moreNodes)
          {
             nodeIds.add(n.getId());
          }

          for (Edge e : links)
          {
             if (!nodeIds.contains(e.outVertex().id().toString()))
             {
                // something was wrong, just add the vertex, for now....
                nodes.add(e.outVertex());
                nodeIds.add(e.outVertex().id().toString());
             }

             if (!nodeIds.contains(e.inVertex().id().toString()))
             {
                // something was wrong, just add the vertex, for now....
                nodes.add(e.inVertex());
                nodeIds.add(e.inVertex().id().toString());
             }

             linkList.add(new CallGraphLink(nextLinkId.getAndIncrement(),
                                            e.outVertex().id().toString(),
                                            e.inVertex().id().toString(),
                                            e.label()));
          }

          for (Vertex v : nodes)
          {
             int    nodeType = v.<Integer>property("node-type").value();
             String label    = describeNode(v, nodeType);
             String vid      = v.id().toString();
             String tooltip  = describeTooltip(v, nodeType);

             nodeList.add(new CallGraphNode(vid, nodeType, label, tooltip));

             CallSite astDet = buildCallSite(v);

             if (astDet != null)
             {
                astDetails.put(vid, astDet);
             }
          }

          nodeList.addAll(moreNodes);
          moreLinks.forEach(l -> { if (!l.getSource().equals(l.getTarget())) linkList.add(l); } );

          Collections.sort(linkList, new Comparator<CallGraphLink>()
          {
             @Override
             public int compare(CallGraphLink o1, CallGraphLink o2)
             {
                return o1.getId() - o2.getId();
             }
          });

          db.tx().commit();

          return new CallGraphSnippet(nodeList, linkList, astDetails, rootList);
       }
    }