CallGraphApi.java
/*
** Module : CallGraphApi.java
** Abstract : Provides an interface for querying, traversing and manipulating the call graph
** from a web client.
**
** Copyright (c) 2017-2023, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------Description----------------------------------
** 001 GES 20170518 First version.
** 002 CA 20180320 Ambiguous report now lists the hint ID.
** A performance improvement for the dependency report.
** CA 20180326 Some improvements for the Dead Code and Dependencies reports.
** 003 CA 20180413 Performance improvement: during report generation, some parents may be cached
** in the vertex properties; if so, we need to commit the tx.
** CA 20180418 Re-write reports to improve performance.
** Dead code now reports all files with possible dead entry points.
** CA 20180508 Ambiguous Call Sites report shows only processed (finished) files.
** CA 20180509 Allow include-level hints to disambiguate call sites.
** CA 20180525 Do not include virtual funcs/procs in the dead code details for a program file.
** 004 CA 20181003 Expose all non-virtual entry points in the callgraph.
** 005 CA 20200428 Some changes to process DEFINE ENUM. Untested.
** 006 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.report.server;
import java.io.*;
import java.util.*;
import java.util.concurrent.atomic.*;
import java.util.function.*;
import java.util.logging.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.uast.*;
import org.apache.tinkerpop.gremlin.structure.*;
import static org.apache.tinkerpop.gremlin.process.traversal.P.*;
/**
* Provides an interface for querying, traversing and manipulating the call graph from a
* web client.
*/
public class CallGraphApi
implements ProgressParserTokenTypes
{
/** Traverse the graph in both upstream and downstream directions. */
public static int BOTH_DIRECTIONS = 0;
/** Traverse the graph in the downstream direction. */
public static int DOWNSTREAM = 1;
/** Traverse the graph in the upstream direction. */
public static int UPSTREAM = 2;
/** Logger. */
private static final ConversionStatus LOG = ConversionStatus.get(CallGraphApi.class);
/** Call graph API base message ID. */
private static final int MSG_BASE = ReportProtocol.MSG_BASE + 2000;
/** Message ID for reading the graph. */
private static final int MSG_READ_GRAPH = MSG_BASE + 1;
/** Message ID for missing targets report. */
private static final int MSG_REPORT_MISSING_TARGETS = MSG_BASE + 2;
/** Message ID for call-sites right-side report. */
private static final int MST_REPORT_CALL_SITES = MSG_BASE + 3;
/** Message ID for external dependencies report. */
private static final int MST_REPORT_EXTERNAL_DEP = MSG_BASE + 4;
/** Message ID for dead code report. */
private static final int MST_REPORT_DEAD_CODE = MSG_BASE + 5;
/** Message ID for ambiguous call-sites report. */
private static final int MST_REPORT_AMBIG_CALLSITES = MSG_BASE + 6;
/** Message ID for right-side ambiguous file's call-sites report. */
private static final int MST_REPORT_FILE_AMBIG_CALLSITES = MSG_BASE + 7;
/** Message ID for right-side ambiguous file's call-sites report. */
private static final int MST_REPORT_DEAD_DEPENDENCIES = MSG_BASE + 8;
/** Message ID for verify schema triggers report. */
private static final int MST_REPORT_VERIFY_SHEMA_TRIGGERS = MSG_BASE + 9;
/** Message ID for dependencies report. */
private static final int MST_REPORT_LIST_DEPENDENCIES = MSG_BASE + 10;
/** Message ID for right-side dependencies report. */
private static final int MST_REPORT_DEPENDENCIES_FOR = MSG_BASE + 11;
/** The callgraph is emitted only for file resources. */
private static final int GRAPH_DETAIL_FILES = 0;
/** The callgraph is emitted for only for entry points. */
private static final int GRAPH_DETAIL_ENTRY_POINTS = 1;
/** The callgraph is emitted with all details. */
private static final int GRAPH_DETAIL_ALL = 2;
/** Graph database instance to use. */
private static Graph db = CallGraphHelper.getGraph();
/**
* Default constructor.
*/
public CallGraphApi()
{
}
/**
* Build the {@link CallSite} instance with the details for the specified callgraph vertex.
*
* @param v
* The callgraph vertex object.
*
* @return The {@link CallSite} instance.
*/
public static CallSite buildCallSite(Vertex v)
{
Vertex vfile = null;
Vertex vAst = null;
String filename = null;
int srcType = ReportApi.SRC_CODE_CACHE;
if (v.<Integer>property("resource-domain").value() == AST_NODE ||
v.<Integer>property("resource-domain").value() == FILE_RESOURCE)
{
vfile = CallGraphHelper.findProcedureFile(v);
if (vfile == null)
{
vfile = CallGraphHelper.findSchemaFile(v);
srcType = ReportApi.SRC_SCHEMA;
if (vfile != null)
{
vAst = db.traversal().V(vfile).out("contains")
.has("node-type", DATABASE).next();
filename = vfile.<String>property("srcfile").value();
}
}
else
{
filename = vfile.<String>property("os-filename").value();
vAst = db.traversal().V(vfile).out("contains")
.has("node-type", EXTERNAL_PROCEDURE).next();
}
}
if (vfile != null)
{
// this is the root node of the file, NOT the actual call site's AST id
long astId = vAst.<Long>property("node-id").value();
CallSite cs = new CallSite();
cs.setFilename(filename);
cs.setLine(v.property("line").isPresent() ? v.<Integer>property("line").value() : 0);
cs.setColumn(v.property("column").isPresent() ? v.<Integer>property("column").value() : 0);
cs.setVertexId(v.id().toString());
cs.setCallSite(describeNode(v, v.<Integer>property("node-type").value()));
cs.setAstId(astId);
cs.setNodeId(v.<Long>property("node-id").value());
cs.setTokenType(v.property("type").isPresent() ? v.<Integer>property("type").value() : 0);
cs.setTokenText(v.property("text").isPresent() ? v.<String>property("text").value() : "");
cs.setSrcType(srcType);
try
{
cs.setFid(ReportApi.getFileId(filename));
}
catch (Throwable t)
{
// ignore
}
return cs;
}
return null;
}
/**
* Calculate the descriptive text for a given vertex.
*
* @param v
* The vertex to describe.
* @param type
* The token type for the node.
*
* @return The descriptive text for the node.
*/
private static String describeNode(Vertex v, int type)
{
String label = null;
int astType = (v.property("type").isPresent() ? v.<Integer>property("type").value() : 0);
if (type == EXTERNAL_PROCEDURE ||
astType == KW_RUN ||
astType == KW_ASYNC ||
astType == COM_METHOD ||
(astType > BEGIN_METH && astType < END_METH) ||
(astType > BEGIN_FUNCTYPES && astType < END_FUNCTYPES))
{
label = ProgressParser.lookupTokenName(type);
}
else if (v.property("os-filename").isPresent())
{
label = (String) CallGraphHelper.property(v, "os-filename");
}
else if (v.property("node-key").isPresent())
{
label = CallGraphHelper.getNodeKeyValue(v);
}
else
{
label = v.label();
}
return label;
}
/**
* Calculate the descriptive text for a given vertex.
*
* @param v
* The vertex to describe.
* @param type
* The token type for the node.
*
* @return The descriptive text for the node.
*/
private static String describeTooltip(Vertex v, int type)
{
String tooltip = null;
int astType = (v.property("type").isPresent() ? v.<Integer>property("type").value() : 0);
if (astType == KW_ON)
{
String events = v.property("events").isPresent() ? v.<String>property("events").value()
: null;
if (events != null)
{
tooltip = events;
}
return tooltip;
}
// only these have custom tooltip for now
if (!(astType == KW_RUN ||
type == EVENT ||
astType == KW_ASYNC ||
type == FUNCTION_CALL ||
type == KW_DYN_FUNC))
{
return null;
}
tooltip = "";
Iterator<Vertex> iter = db.traversal().V(v).out("calls");
while (iter.hasNext())
{
Vertex targetV = iter.next();
int nodeType = targetV.<Integer>property("node-type").value();
Vertex vFile = null;
int vDom = targetV.<Integer>property("resource-domain").value();
if (vDom == AST_NODE)
{
vFile = CallGraphHelper.findProcedureFile(targetV);
}
String txt = "";
if (vFile != null && !vFile.equals(targetV))
{
txt += vFile.<String>property("filename").value();
}
if (nodeType != EXTERNAL_PROCEDURE)
{
if (targetV.<String>property("node-key").isPresent())
{
txt += " " + CallGraphHelper.getNodeKeyValue(targetV);
}
else
{
txt += " " + targetV.label();
}
}
if (targetV.property("line").isPresent())
{
int line = targetV.<Integer>property("line").value();
int column = targetV.<Integer>property("column").value();
String lbl = nodeType == TRIGGER_BLOCK ? "registers" : "contains";
Vertex parentV = targetV;
while (parentV != null && line == 0 && column == 0)
{
Iterator<Vertex> iterP = db.traversal().V(parentV).in(lbl);
if (iterP.hasNext())
{
parentV = iterP.next();
if (parentV.property("line").isPresent())
{
line = parentV.<Integer>property("line").value();
column = parentV.<Integer>property("column").value();
}
}
else
{
parentV = null;
}
}
if (parentV != null && line != 0 && column != 0)
{
txt += "@" + line + ":" + column;
}
}
if (!tooltip.isEmpty())
{
tooltip += "; ";
}
tooltip += txt;
}
tooltip = tooltip.trim();
return tooltip.isEmpty() ? null
: tooltip.indexOf(";") == -1 ? tooltip
: " Calls: " + tooltip.trim();
}
/**
* Obtain a subset of the call graph based on the given criteria.
*
* @param files
* The list of AST filenames to use as the starting point for the graph snippet
* or <code>null</code> to use the project's preconfigured root node list.
* @param profileId
* If greater than 0, then is the ID of a filter profile.
* @param direction
* One of the direction constants from this class for {@link #BOTH_DIRECTIONS},
* {@link #DOWNSTREAM} or {@link #UPSTREAM}. This controls which edges (inbound
* and/or outbound) are included. UPSTREAM maps to inbound and DOWNSTREAM maps
* to outbound.
* @param levels
* This defines the number of cross-file levels of traversal to include in the
* snippet. Since some nodes are contained (using edges labeled "contains") in
* other nodes, using a simple calculation of edge levels would lead to inconsistent
* results. Instead, we track the number of times that the graph traveral crosses
* from one file domain resource into another file domain resource. Use 0 to
* include only the starting nodes, 1 to include the starting nodes plus the nodes
* that can be traversed through 1 edge (in the direction specified), 2 for
* traversal through 2 edges and so forth. -1 means there is no limit. Use the no
* limit options with caution as there may be performance and memory issues with
* the result.
* @param detail
* The level of detail for the graph.
*
* @return The specified subset of the graph.
*/
@WebApi(type = MSG_READ_GRAPH)
public CallGraphSnippet readGraph(String[] files,
long profileId,
int direction,
int levels,
int detail)
{
if (levels == -1)
{
List<Vertex> lf = db.traversal().V().has("node-type", PROCEDURE_FILE).toList();
files = new String[lf.size()];
int i = 0;
for (Vertex v : lf)
{
files[i++] = v.<String>property("os-filename").value() + ".ast";
}
}
else if (profileId > 0)
{
List<String> fl = new ArrayList<>();
// read a file set
for (String f : ReportApi.listFilesFromProfile(profileId))
{
fl.add(f + ".ast");
}
files = new TreeSet<String>(fl).toArray(new String[0]);
}
else if (files == null)
{
try
{
// read root node list from XML
RootNodeList rnl = new RootNodeList();
files = rnl.getList();
}
catch (ConfigurationException | FileNotFoundException e)
{
// fall through
LOG.log(Level.SEVERE, "", e);
}
}
if (files != null)
{
for (int i = 0; i < files.length; i++)
{
if (!files[i].endsWith(".ast"))
{
files[i] += ".ast";
}
}
}
Set<Vertex> nodes = new LinkedHashSet<>();
Set<Edge> links = new LinkedHashSet<>();
Set<CallGraphLink> virtualLinks = new LinkedHashSet<>();
Set<CallGraphNode> moreNodes = new LinkedHashSet<>();
Set<CallGraphLink> moreLinks = new LinkedHashSet<>();
// the next link ID
AtomicInteger nextLinkId = new AtomicInteger(0);
Set<Vertex> start = new LinkedHashSet<>(CallGraphHelper.loadFileNodes(files, true));
if (start != null)
{
// recursively process each level of the graph and store the results
// in the given sets
Map<Vertex, List<Edge>> level0 = traverseLevel(start,
direction,
levels,
detail,
nodes,
links,
virtualLinks,
nextLinkId);
Map<CallGraphLink, CallGraphLink> p2pLinks = new HashMap<>();
// level0 targets need to be processed here, as we need to be sure they are real level0
for (Vertex v : level0.keySet())
{
List<Edge> edges = level0.get(v);
Vertex vParent = computeParentForDetail(v, detail);
if (vParent == null)
{
// not from my level of detail
continue;
}
int vresDom = v.<Integer>property("resource-domain").value();
Vertex vFile = null;
if (vresDom == AST_NODE || vresDom == EVENT)
{
vFile = CallGraphHelper.findProcedureFile(v);
if (vFile == null)
{
vFile = CallGraphHelper.findParentAstVertex(v, SCHEMA_FILE);
}
}
// process each edge
for (Edge e : edges)
{
// don't process links twice
if (links.contains(e))
{
continue;
}
// find the vertext on the other side of the edge
boolean in = e.inVertex().equals(v);
Vertex other = in ? e.outVertex() : e.inVertex();
Vertex otherParent = computeParentForDetail(other, detail);
if (otherParent == null)
{
// not on the detail level
continue;
}
if (detail == GRAPH_DETAIL_ALL)
{
// the node is already part of the graph and we don't need a placeholder
if (nodes.contains(other))
{
links.add(e);
continue;
}
}
else
{
if (nodes.contains(otherParent) && nodes.contains(vParent))
{
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
(!in ? vParent : otherParent).id().toString(),
(in ? vParent : otherParent).id().toString(),
e.label()));
continue;
}
}
// here, we need to determine if the target is a MORE node or not.
// a MORE node can be only a PROCEDURE_FILE or a SCHEMA_FILE, and only if the
// vertices in this edge are not in the same file
// determine the node's file
int oresDom = other.<Integer>property("resource-domain").value();
Vertex ovFile = null;
if (oresDom == AST_NODE || oresDom == EVENT)
{
ovFile = CallGraphHelper.findProcedureFile(other);
if (ovFile == null)
{
ovFile = CallGraphHelper.findParentAstVertex(other, SCHEMA_FILE);
}
}
if (ovFile != null && vFile != null)
{
if (ovFile.id().equals(vFile.id()))
{
if (detail == GRAPH_DETAIL_ALL)
{
// in the same file, just add them
links.add(e);
nodes.add(other);
}
else
{
nodes.add(otherParent);
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
(!in ? vParent : otherParent).id().toString(),
(in ? vParent : otherParent).id().toString(),
e.label()));
}
}
else
{
// transform it into a MORE node and create a link to it, from file to file
String otherId = ovFile.id().toString();
int atype = ovFile.<Integer>property("node-type").value();
String text = describeNode(ovFile, atype);
String tooltip = describeTooltip(ovFile, atype);
moreNodes.add(new CallGraphNode(otherId, MORE, text, tooltip));
String srcID = in ? otherId : vFile.id().toString();
String tarID = in ? vFile.id().toString() : otherId;
CallGraphLink parentLink = new CallGraphLink(nextLinkId.get(),
srcID,
tarID,
e.label());
if (p2pLinks.containsKey(parentLink))
{
parentLink = p2pLinks.get(parentLink);
}
else
{
// create a link to it
moreLinks.add(parentLink);
p2pLinks.put(parentLink, parentLink);
nextLinkId.incrementAndGet();
}
// for this inter-program link, create an explicit link and set its parent
// now
String vId = (detail == GRAPH_DETAIL_ALL ? v : vParent).id().toString();
moreLinks.add(new CallGraphLink(nextLinkId.get(),
parentLink.getId(),
in ? otherId : vId,
in ? vId : otherId,
e.label()));
nextLinkId.incrementAndGet();
}
}
else
{
// in this case, the other side of the link is something else (i.e. an
// include file, an external resource, etc). so, just link to it
if (detail == GRAPH_DETAIL_ALL)
{
nodes.add(other);
links.add(e);
}
else
{
nodes.add(otherParent);
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
(!in ? vParent : otherParent).id().toString(),
(in ? vParent : otherParent).id().toString(),
e.label()));
}
}
}
}
moreLinks.addAll(virtualLinks);
}
return createSnippet(nextLinkId, nodes, links, moreNodes, moreLinks, files);
}
/**
* List all the call-sites targeting this vertex from the callgraph.
*
* @param vertexId
* The vertex ID.
*
* @return Details about all the call-sites incoming in this vertex.
*/
@WebApi(type = MST_REPORT_CALL_SITES)
public List<CallSite> getCallSites(String vertexId)
{
List<CallSite> res = new ArrayList<>();
Iterator<Vertex> iter = db.traversal()
.V(Long.parseLong(vertexId))
.inE("calls", "references")
.outV();
while (iter.hasNext())
{
Vertex out = iter.next();
CallSite cs = buildCallSite(out);
if (cs != null)
{
res.add(cs);
}
}
db.tx().commit();
return res;
}
/**
* Get the external targets marked as missing.
*
* @return The details for the missing targets.
*/
@WebApi(type = MSG_REPORT_MISSING_TARGETS)
public List<CallGraphReportLine> getMissingExternalTargets()
{
Iterator<Map<Object, Long>> iter = db.traversal().V()
.has("external", true)
.has("missing", true)
.inE("calls", "references")
.groupCount().by(e -> ((Edge) e).inVertex().id());
Map<Object, Long> m = iter.hasNext() ? iter.next() : Collections.emptyMap();
List<CallGraphReportLine> res = new ArrayList<>();
Iterator<Object> viter = m.keySet().iterator();
while (viter.hasNext())
{
Object vid = viter.next();
Long count = m.get(vid);
Vertex v = db.traversal().V(vid).next();
String type = v.<String>property("node-key").value();
switch (type)
{
case "filename":
type = "External Program";
break;
case "missing_function":
type = "Function";
break;
case "missing_procedure":
type = "Internal Procedure";
break;
}
CallGraphReportLine mct = new CallGraphReportLine();
mct.setType(type);
mct.setName(CallGraphHelper.getNodeKeyValue(v));
mct.setCallNum(count.intValue());
mct.setVertexId(v.id().toString());
res.add(mct);
}
Collections.sort(res, new Comparator<CallGraphReportLine>()
{
@Override
public int compare(CallGraphReportLine c1, CallGraphReportLine c2)
{
if (c1.getType().equals(c2.getType()))
{
return c1.getName().compareTo(c2.getName());
}
return c1.getType().compareTo(c2.getType());
}
});
db.tx().commit();
return res;
}
/**
* Compute the external dependencies, which are not marked as missing.
*
* @return The report, as a list of external dependencies.
*/
@WebApi(type = MST_REPORT_EXTERNAL_DEP)
public List<CallGraphReportLine> getExternalDependencies()
{
Iterator<Map<Object, Long>> iter = db.traversal().V()
.has("external", true)
.hasNot("missing")
.inE("calls", "references")
.groupCount().by(e -> ((Edge) e).inVertex().id());
Map<Object, Long> m = iter.hasNext() ? iter.next() : Collections.emptyMap();
List<CallGraphReportLine> res = new ArrayList<>();
Iterator<Object> viter = m.keySet().iterator();
while (viter.hasNext())
{
Object vid = viter.next();
Long count = m.get(vid);
Vertex v = db.traversal().V(vid).next();
int nodeType = v.<Integer>property("node-type").value();
String nodeKey = v.<String>property("node-key").value();
String type = v.label();
if (nodeType == NATIVE_API)
{
Vertex vLibRef = v.edges(Direction.IN, "contains").next().outVertex();
type = vLibRef.label() + " - " + CallGraphHelper.getNodeKeyValue(vLibRef);
}
CallGraphReportLine mct = new CallGraphReportLine();
mct.setType(type);
mct.setName(v.<String>property(nodeKey).value());
mct.setCallNum(count.intValue());
mct.setVertexId(v.id().toString());
res.add(mct);
}
Collections.sort(res, new Comparator<CallGraphReportLine>()
{
@Override
public int compare(CallGraphReportLine c1, CallGraphReportLine c2)
{
if (c1.getType().equals(c2.getType()))
{
return c1.getName().compareTo(c2.getName());
}
return c1.getType().compareTo(c2.getType());
}
});
db.tx().commit();
return res;
}
/**
* For a given vertex ID assumed as dead code, return all the vertex ASTs which are marked
* as 'contains' or 'includes', directly from this vertex.
*
* @param vertexId
* The vertex ID.
*
* @return The report details.
*/
@WebApi(type = MST_REPORT_DEAD_DEPENDENCIES)
public List<CallSite> getDeadDependencies(String vertexId)
{
List<CallSite> res = new ArrayList<>();
Iterator<Vertex> iter = db.traversal()
.V(Long.parseLong(vertexId))
.outE("contains", "includes")
.inV()
.hasNot("reachable")
.hasNot("virtual");
while (iter.hasNext())
{
Vertex in = iter.next();
CallSite callSite = buildCallSite(in);
if (callSite == null)
{
CallSite cs = new CallSite();
int nodeType = in.<Integer>property("node-type").value();
cs.setTokenText(ProgressParser.lookupTokenName(nodeType));
cs.setCallSite(describeNode(in, nodeType));
if (nodeType == INCLUDE_FILE)
{
in.edges(Direction.IN, "includes").forEachRemaining(e ->
{
if (e.outVertex().id().toString().equals(vertexId) && e.inVertex().equals(in))
{
int column = e.<Integer>property("column").value();
int line = e.<Integer>property("line").value();
cs.setColumn(column);
cs.setLine(line);
}
});
}
callSite = cs;
}
res.add(callSite);
}
Collections.sort(res, new Comparator<CallSite>()
{
@Override
public int compare(CallSite o1, CallSite o2)
{
return o1.getTokenText().equals(o2.getTokenText())
? o1.getCallSite().compareTo(o2.getCallSite())
: o1.getTokenText().compareTo(o2.getTokenText());
}
});
db.tx().commit();
return res;
}
/**
* Report all dead code entry-points: programs, internal procedures/functions, trigger blocks,
* OO classes, properties and methods.
*
* @return The report details.
*/
@WebApi(type = MST_REPORT_DEAD_CODE)
public List<DeadCodeReportLine> getDeadCode()
{
long start = System.currentTimeMillis();
List<DeadCodeReportLine> res = new ArrayList<>();
Set<Object> reached = new HashSet<>();
String parentProp = CallGraphHelper.parentProperty(PROCEDURE_FILE);
Consumer<Vertex> collect = vk ->
{
if (vk.property("virtual").isPresent())
{
return;
}
VertexProperty<Integer> vp = vk.<Integer>property(parentProp);
Vertex v = vk;
if (vp.isPresent())
{
v = db.traversal().V(vp.value()).next();
}
if (reached.contains(v))
{
return;
}
reached.add(v);
VertexProperty<String> vpFile = v.<String>property("filename");
String pfile = vpFile.isPresent() ? vpFile.value() : CallGraphHelper.getNodeKeyValue(v);
int resDom = v.<Integer>property("resource-domain").value();
int nodeType = v.<Integer>property("node-type").value();
DeadCodeReportLine dc = new DeadCodeReportLine();
dc.setFile(pfile);
dc.setType(ProgressParser.lookupTokenName(resDom == AST_NODE ? PROCEDURE_FILE : nodeType));
dc.setVertexId(v.id().toString());
res.add(dc);
};
// get all nodes with no "reachable" property, of the specified types
db.traversal().V()
.has("node-type",
within(FUNCTION,
INTERNAL_PROCEDURE,
METHOD_DEF,
DEFINE_PROPERTY_SET,
DEFINE_PROPERTY_GET,
CONSTRUCTOR,
DESTRUCTOR,
CLASS_DEF,
INTERFACE_DEF,
ENUM_DEF,
EXTERNAL_PROCEDURE,
PROCEDURE_FILE))
.has("resource-domain", AST_NODE)
.has("reachable", neq(true))
.forEachRemaining(v -> collect.accept(v));
// get all nodes with no "reachable" property, of the specified types
db.traversal().V()
.has("node-type", INCLUDE_FILE)
.has("resource-domain", FILE_RESOURCE)
.has("reachable", neq(true))
.forEachRemaining(v -> collect.accept(v));
// get all nodes with no "reachable" property, of the specified types
db.traversal().V()
.has("external", true)
.has("missing", neq(true))
.has("reachable", neq(true))
.forEachRemaining(v -> collect.accept(v));
Collections.sort(res, new Comparator<DeadCodeReportLine>()
{
@Override
public int compare(DeadCodeReportLine c1, DeadCodeReportLine c2)
{
if (c1.getType().equals(c2.getType()))
{
return c1.getFile().compareTo(c2.getFile());
}
else
{
return c1.getType().compareTo(c2.getType());
}
}
});
long end = System.currentTimeMillis();
System.out.println(String.format("Dead code done in %d millis", end - start));
db.tx().commit();
return res;
}
/**
* Get all the program files with ambiguous call-sites.
*
* @return The list of program files with ambiguous call-sites.
*/
@WebApi(type = MST_REPORT_AMBIG_CALLSITES)
public List<AmbigFileReportLine> getAmbiguousFiles()
{
Vertex vAmbig = CallGraphHelper.findNodeById(AMBIGUOUS, CallGraphHelper.AMBIGUOUS_NODE_ID);
Iterator<Map<Object, Long>> iter = db.traversal().V(vAmbig).inE().outV()
.groupCount()
.by(CallGraphHelper.parentProperty(PROCEDURE_FILE));
Map<Object, Long> groups = iter.hasNext() ? iter.next() : Collections.emptyMap();
Iterator<Object> iter2 = groups.keySet().iterator();
Map<String, Integer> refs = new HashMap<>();
while (iter2.hasNext())
{
Long fileId = (Long) iter2.next();
if (!db.traversal().V(fileId).has("finished", true).hasNext())
{
// show only files which have been processed by the call graph
continue;
}
String filename = (String) db.traversal().V(fileId).properties("filename").value().next();
if (!refs.containsKey(filename))
{
refs.put(filename, 0);
}
Long count = groups.get(fileId);
refs.put(filename, refs.get(filename) + count.intValue());
}
List<AmbigFileReportLine> res = new ArrayList<>();
for (String filename : new TreeSet<String>(refs.keySet()))
{
res.add(new AmbigFileReportLine(filename, refs.get(filename)));
}
db.tx().commit();
return res;
}
/**
* Get all the ambiguous call-sites in the specified program file.
*
* @param filename
* The program file name.
*
* @return The list of ambiguous call-sites.
*/
@WebApi(type = MST_REPORT_FILE_AMBIG_CALLSITES)
public List<CallSite> getFileAmbiguousCallSites(String filename)
{
Vertex vAmbig = CallGraphHelper.findNodeById(AMBIGUOUS, CallGraphHelper.AMBIGUOUS_NODE_ID);
List<CallSite> res = new ArrayList<>();
Vertex vfile = db.traversal().V().has("filename", filename).next();
String procFileProp = CallGraphHelper.parentProperty(PROCEDURE_FILE);
Iterator<Vertex> iter = db.traversal().V(vAmbig).inE().outV().has(procFileProp, vfile.id());
while (iter.hasNext())
{
Vertex v = iter.next();
CallSite cs = buildCallSite(v);
if (cs != null)
{
Iterator<Edge> eiter = v.edges(Direction.OUT, "ambiguous");
if (eiter.hasNext())
{
int hintId = (Integer) CallGraphHelper.property(v, "hint-instance");
String tokType = ProgressParser.lookupTokenName(
v.<Integer>property("node-type").value());
String hint = String.format("%s_%s", tokType, hintId);
cs.setHint(hint);
cs.setFilename(filename + ".hints");
res.add(cs);
Integer includeHintId =
(Integer) CallGraphHelper.property(v, "include-hint-instance");
if (includeHintId != null)
{
String includeHintFile = (String) CallGraphHelper.property(v, "include-hint-file");
String includeHint = String.format("%s_%s", tokType, includeHintId);
cs = new CallSite(cs);
cs.setFilename(includeHintFile + ".hints");
cs.setHint(includeHint);
res.add(cs);
}
}
else
{
System.out.println("Warning: ambiguous node linked with an incorrect edge");
v.properties()
.forEachRemaining(vp -> System.out.println(vp.key() + " " + vp.value()));
v.edges(Direction.BOTH)
.forEachRemaining(e ->
{
System.out.println(e);
e.properties()
.forEachRemaining(vp -> System.out.println(vp.key() + " " + vp.value()));
});
}
}
}
Collections.sort(res, new Comparator<CallSite>()
{
@Override
public int compare(CallSite c1, CallSite c2)
{
if (c1.getFilename().equals(c2.getFilename()))
{
return c1.getLine() == c2.getLine() ? c1.getColumn() - c2.getColumn()
: c1.getLine() - c2.getLine();
}
else
{
return c1.getFilename().compareTo(c2.getFilename());
}
}
});
db.tx().commit();
return res;
}
/**
* Verify the schema triggers loaded in the call graph. This includes the following problems:
* <ol>
* <li>if a schema trigger links to an external program with no TRIGGER_PROCEDURE</li>
* <li>if trigger types don't match</li>
* <li>if the schema name differs</li>
* <li>if it links to a missing external program</li>
* <li>if external procedure with a TRIGGER_PROCEDURE is not linked to any table</li>
* </ol>
*
* @return The report results.
*/
@WebApi(type = MST_REPORT_VERIFY_SHEMA_TRIGGERS)
public List<CallGraphReportLine> getReportVerifySchemaTriggers()
{
List<CallGraphReportLine> res = new ArrayList<>();
// go through all links from a TABLE trigger and report if:
// 1. it links to an external program with no TRIGGER_PROCEDURE specification
// 2. if the trigger type matches
// 3. if the schemaname differs
// 4. if it links to a missing external program
// go through all TRIGGER_PROCEDURE vertices and report if:
// 45. this trigger program is not referenced by any TABLE triggers
String type1 = "External program is not a table trigger";
String type2 = "Trigger type mismatch";
String type3 = "Different table";
String type4 = "Missing external program";
String type5 = "Dead table trigger";
Iterator<Edge> iterE = db.traversal().V().has("node-type", KW_TAB_TRG).outE("calls");
while (iterE.hasNext())
{
Edge e = iterE.next();
Vertex vTabTrig = e.outVertex();
Vertex vTable = vTabTrig.vertices(Direction.IN, "contains").next();
Vertex vExtProg = e.inVertex();
if (vExtProg.property("missing").isPresent())
{
CallGraphReportLine line = new CallGraphReportLine();
line.setVertexId(vExtProg.id().toString());
line.setType(type4);
line.setName(vExtProg.<String>property("filename").value());
res.add(line);
continue;
}
Vertex vTrigProc = CallGraphHelper.findAdjacent(vExtProg,
TRIGGER_PROCEDURE,
"contains",
true);
if (vTrigProc == null)
{
CallGraphReportLine line = new CallGraphReportLine();
line.setVertexId(vExtProg.id().toString());
line.setType(type1);
line.setName(vExtProg.<String>property("external-procedure").value());
res.add(line);
continue;
}
int tableTrigType = e.<Integer>property("trigger-type").value();
int progTrigType = vTrigProc.<Integer>property("trigger-type").value();
if (tableTrigType != progTrigType)
{
CallGraphReportLine line = new CallGraphReportLine();
line.setVertexId(vExtProg.id().toString());
line.setType(type2);
line.setName(vExtProg.<String>property("external-procedure").value());
res.add(line);
}
String tableSchema = vTable.<String>property("schemaname").value();
String progTableSchema = vTrigProc.<String>property("trigger-schema").value();
if (!tableSchema.equals(progTableSchema))
{
CallGraphReportLine line = new CallGraphReportLine();
line.setVertexId(vExtProg.id().toString());
line.setType(type3);
line.setName(vExtProg.<String>property("external-procedure").value());
res.add(line);
}
}
Iterator<Vertex> iterV = db.traversal().V().has("node-type", TRIGGER_PROCEDURE);
while (iterV.hasNext())
{
Vertex vTrigProc = iterV.next();
Vertex vExtProg = vTrigProc.edges(Direction.IN, "contains").next().outVertex();
boolean hasTableTrigger = false;
Iterator<Edge> iterT = vExtProg.edges(Direction.IN, "calls");
while (iterT.hasNext())
{
Edge e = iterT.next();
Vertex vOut = e.outVertex();
if (vOut.<Integer>property("node-type").value() == KW_TAB_TRG)
{
hasTableTrigger = true;
break;
}
}
if (!hasTableTrigger)
{
CallGraphReportLine line = new CallGraphReportLine();
line.setVertexId(vExtProg.id().toString());
line.setType(type5);
line.setName(vExtProg.<String>property("external-procedure").value());
res.add(line);
}
}
Collections.sort(res, new Comparator<CallGraphReportLine>()
{
@Override
public int compare(CallGraphReportLine l1, CallGraphReportLine l2)
{
return l1.getType().equals(l2.getType())
? l1.getName().compareTo(l2.getName())
: l1.getType().compareTo(l2.getType());
}
});
db.tx().commit();
return res;
}
/**
* List all the dependencies for the entire program file list.
* <p>
* Left side will include the entire loaded code set (which are
* {@link ProgressParserTokenTypes#FILE_RESOURCE file-level entry points}) and right-side will
* include the targets to other (non-internal) nodes.
*
* @return The report results.
*/
@WebApi(type = MST_REPORT_LIST_DEPENDENCIES)
public List<CallGraphReportLine> getDependenciesReport()
{
List<CallGraphReportLine> res = new ArrayList<>();
db.traversal().V()
.has("resource-domain", FILE_RESOURCE)
.has("os-filename")
.forEachRemaining(v ->
{
// avoid this code, as the total time of the report increases
// int outDependencies = getDependenciesFor(v.id().toString(), true).size();
// if (outDependencies == 0)
// {
// continue;
// }
CallGraphReportLine line = new CallGraphReportLine();
line.setVertexId(v.id().toString());
// line.setCallNum(outDependencies);
line.setName(v.<String>property("os-filename").value());
line.setType(v.label());
res.add(line);
});
Collections.sort(res, new Comparator<CallGraphReportLine>()
{
@Override
public int compare(CallGraphReportLine l1, CallGraphReportLine l2)
{
return l1.getType().equals(l2.getType())
? l1.getName().compareTo(l2.getName())
: l1.getType().compareTo(l2.getType());
}
});
db.tx().commit();
return res;
}
/**
* Get the dependencies for the specified file resources (as vertex ID).
*
* @param vertexId
* The vertex ID.
*
* @return The details about the remote call-sites.
*/
@WebApi(type = MST_REPORT_DEPENDENCIES_FOR)
public List<CallSite> getDependenciesFor(String vertexId)
{
List<CallSite> res = getDependenciesFor(vertexId, false);
db.tx().commit();
return res;
}
/**
* Get the dependencies for the specified file resources (as vertex ID).
*
* @param vertexId
* The vertex ID.
* @param internal
* When this flag is set, only first found dependency is reported.
*
* @return The details about the remote call-sites.
*/
private List<CallSite> getDependenciesFor(String vertexId, boolean internal)
{
List<CallSite> res = new ArrayList<>();
Set<Vertex> nodes = new LinkedHashSet<>();
Set<Edge> links = new LinkedHashSet<>();
Vertex v = db.traversal().V(vertexId).next();
int nodeType = v.<Integer>property("node-type").value();
boolean isInclude = nodeType == INCLUDE_FILE || v.edges(Direction.IN, "includes").hasNext();
if (isInclude)
{
// include files report only which program files reference them...
Iterator<Edge> iterE = v.edges(Direction.IN, "includes");
while (iterE.hasNext())
{
Edge e = iterE.next();
Vertex vOut = e.outVertex();
Vertex vFile = null;
if (e.property("call-site-id").isPresent())
{
long parentId = e.<Long>property("call-site-id").value();
vFile = CallGraphHelper.findNodeById(PROCEDURE_FILE, parentId);
}
else if (vOut.property("filename").isPresent())
{
vFile = vOut;
}
else
{
continue;
}
CallSite csOut = new CallSite();
csOut.setFilename("included by " + CallGraphHelper.getNodeKeyValue(vFile));
csOut.setCallSite(CallGraphHelper.getNodeKeyValue(vOut));
if (e.property("start-line").isPresent())
{
csOut.setLine(e.<Integer>property("start-line").value());
csOut.setColumn(e.<Integer>property("start-column").value());
}
else if (vOut.property("line").isPresent())
{
csOut.setLine(vOut.<Integer>property("line").value());
csOut.setColumn(vOut.<Integer>property("column").value());
}
res.add(csOut);
if (internal)
{
return res;
}
}
}
else
{
Vertex vp = computeParentForDetail(v, GRAPH_DETAIL_FILES);
traverseFileResource(v, nodes, links);
for (Vertex nOut : nodes)
{
Iterator<Edge> iterE = nOut.edges(Direction.OUT);
while (iterE.hasNext())
{
Edge e = iterE.next();
// skip the INCLUDES edges
if (e.label().equals("includes"))
{
continue;
}
Vertex nIn = e.inVertex();
// find its parent...
Vertex nInParent = computeParentForDetail(nIn, GRAPH_DETAIL_FILES);
Vertex nOutParent = computeParentForDetail(nOut, GRAPH_DETAIL_FILES);
if (nOutParent == null || !nOutParent.equals(vp))
{
continue;
}
if (nInParent != null && !vp.equals(nInParent))
{
CallSite csOut = buildCallSite(nOut);
if (csOut == null)
{
continue;
}
String descr = null;
if (nInParent.property("os-filename").isPresent())
{
descr = nInParent.<String>property("os-filename").value();
}
else
{
descr = CallGraphHelper.getNodeKeyValue(nInParent);
}
String nodeKey = nIn.<String>property("node-key").value();
if (!nIn.equals(nInParent) &&
!(nodeKey.equals("filename") ||
nodeKey.equals("os-filename") ||
nodeKey.equals("external-procedure")))
{
descr = descr + " " + nIn.<String>property(nodeKey).value();
}
descr = e.label() + " " + nIn.label() + " (" + descr + ")";
csOut.setFilename(descr);
if (e.property("line").isPresent())
{
csOut.setLine(e.<Integer>property("line").value());
}
if (e.property("column").isPresent())
{
csOut.setColumn(e.<Integer>property("column").value());
}
res.add(csOut);
if (internal)
{
return res;
}
}
}
}
// do a DFS search and get the include tree; start with the vertices targeted by
// INCLUDES edges from the vp node.
Queue<Edge> inext = new ArrayDeque<>();
vp.edges(Direction.OUT, "includes").forEachRemaining(edge -> inext.add(edge));
while (!inext.isEmpty())
{
Edge e = inext.poll();
Vertex vOut = e.outVertex();
Vertex vIn = e.inVertex();
CallSite csOut = new CallSite();
csOut.setFilename("includes " + CallGraphHelper.getNodeKeyValue(vIn));
csOut.setCallSite(CallGraphHelper.getNodeKeyValue(vOut));
if (e.property("start-line").isPresent())
{
csOut.setLine(e.<Integer>property("start-line").value());
csOut.setColumn(e.<Integer>property("start-column").value());
}
else if (vOut.property("line").isPresent())
{
csOut.setLine(vOut.<Integer>property("line").value());
csOut.setColumn(vOut.<Integer>property("column").value());
}
res.add(csOut);
if (internal)
{
return res;
}
vIn.edges(Direction.OUT, "includes").forEachRemaining(edge ->
{
Property<String> p = edge.<String>property("parent-edge-id");
if (p.isPresent() && p.value().equals(e.id().toString()))
{
inext.add(edge);
}
});
}
}
Collections.sort(res, new Comparator<CallSite>()
{
@Override
public int compare(CallSite c1, CallSite c2)
{
if (c1.getFilename().equals(c2.getFilename()))
{
return c1.getLine() == c2.getLine() ? c1.getColumn() - c2.getColumn()
: c1.getLine() - c2.getLine();
}
else
{
return c1.getFilename().compareTo(c2.getFilename());
}
}
});
return res;
}
/**
* Traverse the file resource and load the entire "contains" or "includes" tree.
*
* @param start
* The node to start with.
* @param nodes
* The vertex collector.
* @param links
* The link collector.
*/
private void traverseFileResource(Vertex start, Set<Vertex> nodes, Set<Edge> links)
{
if (nodes.contains(start))
{
return;
}
nodes.add(start);
String[] lbl = start.<Integer>property("node-type").value() == INCLUDE_FILE
? new String[] { "includes" }
: new String[] { "contains", "raises", "listens", "registers" };
Iterator<Edge> iter = db.traversal().V(start).outE(lbl);
while (iter.hasNext())
{
Edge nextE = iter.next();
Vertex nextV = nextE.inVertex();
links.add(nextE);
traverseFileResource(nextV, nodes, links);
}
}
/**
* Gather the vertices and edges for the specified portion of the graph and optionally
* use recursion to get each additional requested level until all levels are gathered.
*
* @param start
* The list of vertices to use as the starting point.
* @param direction
* One of the direction constants from this class for {@link #BOTH_DIRECTIONS},
* {@link #DOWNSTREAM} or {@link #UPSTREAM}. This controls which edges (inbound
* and/or outbound) are included. UPSTREAM maps to inbound and DOWNSTREAM maps
* to outbound.
* @param levels
* This defines the number of cross-file levels of traversal to include in the
* snippet. Since some nodes are contained (using edges labeled "contains") in
* other nodes, using a simple calculation of edge levels would lead to inconsistent
* results. Instead, we track the number of times that the graph traveral crosses
* from one file domain resource into another file domain resource. Use 0 to
* include only the starting nodes, 1 to include the starting nodes plus the nodes
* that can be traversed through 1 edge (in the direction specified), 2 for
* traversal through 2 edges and so forth. -1 means there is no limit. Use the no
* limit options with caution as there may be performance and memory issues with
* the result.
* @param detail
* The level of detail for the graph.
* @param nodes
* The set in which to gather the vertices.
* @param links
* The set in which to gather the edges.
* @param virtualLinks
* For a graph detail other than {@link #GRAPH_DETAIL_ALL}, it will contain virtual
* links between the vertices which are not expanded.
* @param nextLinkId
* The counter which generates link IDs.
*
* @return The level-0 assumed edges, per each vertex. These edges will be processed against
* the final graph, to determine if they are real level-0 or not.
*/
private Map<Vertex, List<Edge>> traverseLevel(Set<Vertex> start,
int direction,
int levels,
int detail,
Set<Vertex> nodes,
Set<Edge> links,
Set<CallGraphLink> virtualLinks,
AtomicInteger nextLinkId)
{
if (start.isEmpty())
{
return Collections.emptyMap();
}
Map<Vertex, List<Edge>> level0 = new LinkedHashMap<>();
// make sure that these nodes are added to the graph
nodes.addAll(start);
String[] edgeLbls = { "calls", "raises", "listens", "registers", "references" };
Set<Vertex> newNodes = new LinkedHashSet<>();
// for each vertex in the starting list
for (Vertex s : start)
{
Set<Vertex> contained = new LinkedHashSet<Vertex>();
Set<Edge> containedLinks = new LinkedHashSet<Edge>();
traverseFileResource(s, contained, containedLinks);
// the above will produce the entire sub-graph for this file; from these, keep only nodes
// which are involved in the callgraph ('calls', 'listens' and 'raises'), incoming or
// outgoing
Set<Vertex> keepNodes = new HashSet<>();
Set<Edge> keepLinks = new HashSet<>();
for (Vertex v : contained)
{
VertexProperty<Boolean> vEntryPoint = v.property("entry-point");
if (vEntryPoint.isPresent() && vEntryPoint.value() && !v.property("virtual").isPresent())
{
// always keep all defined entry points
keepNodes.add(v);
v.edges(Direction.IN, "contains").forEachRemaining(e -> keepLinks.add(e));
}
Iterator<Edge> iter = v.edges(Direction.BOTH, edgeLbls);
if (iter.hasNext())
{
Vertex vp = v;
while (vp != null)
{
keepNodes.add(vp);
Iterator<Edge> iter2 = db.traversal().V(vp).inE("contains");
if (iter2.hasNext())
{
Edge ep = iter2.next();
keepLinks.add(ep);
vp = ep.outVertex();
}
else
{
vp = null;
}
}
// these are all in the current context
String[] lbls = { "raises", "listens", "registers" };
for (String lbl : lbls)
{
Iterator<Edge> iter2 = db.traversal().V(v).bothE(lbl);
if (iter2.hasNext())
{
Edge ep = iter2.next();
keepLinks.add(ep);
keepNodes.add(ep.outVertex());
keepNodes.add(ep.inVertex());
}
}
}
}
{
// the 'contained' sets must remain with the 'keep' sets, but preserving order... so
// this difference quirk is used
Set<Vertex> s1 = new HashSet<>(contained);
s1.removeAll(keepNodes);
contained.removeAll(s1);
Set<Edge> s2 = new HashSet<>(containedLinks);
s2.removeAll(keepLinks);
containedLinks.removeAll(s2);
}
// save the links...
if (detail == GRAPH_DETAIL_ALL)
{
links.addAll(containedLinks);
}
else if (detail == GRAPH_DETAIL_ENTRY_POINTS)
{
for (Edge e : containedLinks)
{
Vertex vIn = e.inVertex();
Vertex vOut = e.outVertex();
Vertex vTargetIn = computeParentForDetail(vIn, detail);
Vertex vTargetOut = computeParentForDetail(vOut, detail);
// for this case, keep only the intra-links which reach the detail level (i.e.
// 'contains' links from program file to function, internal proc, etc
if (vIn.equals(vTargetIn) && vOut.equals(vTargetOut))
{
newNodes.add(vTargetOut);
newNodes.add(vTargetIn);
// the link is a 'contains' intra-link with the proper detail level, keep it
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
vTargetOut.id().toString(),
vTargetIn.id().toString(),
e.label()));
}
}
}
else
{
// for the GRAPH_DETAIL_FILES, the 'contained' links are not inter-files links, so
// do not add anything
}
// process all of the contained nodes for the current node in our starting list
for (Vertex v : contained)
{
// don't process nodes twice
if (nodes.contains(v))
{
continue;
}
Vertex parent = computeParentForDetail(v, detail);
if (parent != null)
{
// save the parent node (as this is the detail level)
nodes.add(parent);
}
else
{
// parent can't be found, so what to do?
// TODO: !!!!!!!!!
continue;
}
List<Edge> edges = null;
List<Vertex> next = null;
// gather the non-contains edges from those contained nodes
if (direction == DOWNSTREAM)
{
edges = db.traversal().V(v).outE(edgeLbls).toList();
next = db.traversal().V(v).out(edgeLbls).toList();
}
else if (direction == UPSTREAM)
{
edges = db.traversal().V(v).inE(edgeLbls).toList();
next = db.traversal().V(v).in(edgeLbls).toList();
}
else
{
// otherwise assume BOTH_DIRECTIONS
edges = db.traversal().V(v).bothE(edgeLbls).toList();
next = db.traversal().V(v).both(edgeLbls).toList();
}
// edges might point to external resources which are not single-node (i.e. native
// API calls, missing external procedures/internal procedures/functions). These
// cases can't be treated as MORE nodes (as their target is an actual program file)
// so they need to be expanded fully here
Iterator<Edge> iterE = edges.iterator();
while (iterE.hasNext())
{
Edge e = iterE.next();
Vertex vOther = (v.equals(e.inVertex()) ? e.outVertex() : e.inVertex());
if (vOther.property("external").isPresent() &&
vOther.<Boolean>property("external").value())
{
next.remove(vOther);
iterE.remove();
Iterator<Edge> iterEContains = vOther.edges(Direction.IN, "contains");
if (iterEContains.hasNext())
{
// there is only one level of contains here. get the parent and add all its
// children, to have a complete state
Edge eContains = iterEContains.next();
Vertex vOtherParent = eContains.outVertex();
nodes.add(vOtherParent);
if (detail == GRAPH_DETAIL_FILES)
{
// create a 'to-file' link and hide its internal representation
Vertex vParent = CallGraphHelper.findProcedureFile(v);
if (vParent == null)
{
vParent = CallGraphHelper.findSchemaFile(v);
}
nodes.add(vParent);
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
(v.equals(e.outVertex()) ? vParent.id() : vOtherParent.id()).toString(),
(v.equals(e.inVertex()) ? vParent.id() : vOtherParent.id()).toString(),
e.label()));
}
else
{
// just add the link, is not hidden
links.add(e);
iterEContains = vOtherParent.edges(Direction.OUT, "contains");
while (iterEContains.hasNext())
{
eContains = iterEContains.next();
links.add(eContains);
nodes.add(eContains.inVertex());
}
}
}
else
{
Vertex vOut = e.outVertex();
Vertex vIn = e.inVertex();
Vertex vTargetOut = computeParentForDetail(vOut, detail);
Vertex vTargetIn = computeParentForDetail(vIn, detail);
if (vTargetOut != null && vTargetIn != null && !vTargetOut.equals(vTargetIn))
{
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
vTargetOut.id().toString(),
vTargetIn.id().toString(),
e.label()));
nodes.add(vTargetOut);
nodes.add(vTargetIn);
}
}
}
}
if (levels == 0)
{
// on the depth-first search being done here, although an edge might look like it
// points outside of our max-level structure, another parallel path might meet the
// same 'remote' vertex sooner, so this node can't become a MORE node; for this
// reason, we collect 'level0' edges and process them only after the entire DFS
// has finished
if (!edges.isEmpty())
{
level0.put(v, edges);
}
}
else
{
Set<Vertex> nextStart = new LinkedHashSet<>();
for (Vertex n : next)
{
int nodeType = n.<Integer>property("node-type").value();
int resDom = n.<Integer>property("resource-domain").value();
if (resDom == AST_NODE || resDom == EVENT)
{
Vertex pNode = CallGraphHelper.findProcedureFile(n);
if (pNode != null)
{
nextStart.add(pNode);
}
else
{
pNode = CallGraphHelper.findSchemaFile(n);
if (pNode != null)
{
nextStart.add(pNode);
}
}
}
/*
else if (nodeType == INCLUDE_FILE)
{
nextStart.add(n);
}
*/
else if (n.property("external").isPresent() &&
n.<Boolean>property("external").value())
{
Vertex pNode = n;
if (nodeType == NATIVE_API ||
nodeType == INTERNAL_PROCEDURE ||
nodeType == FUNCTION)
{
// find the parent container for a native API (shared library) and for
// missing internal proc/function (the external program, missing or not)
Iterator<Vertex> iter = n.vertices(Direction.IN, "contains");
pNode = (iter.hasNext() ? iter.next() : null);
}
if (pNode != null)
{
nextStart.add(pNode);
}
}
}
// remove the already walked nodes...
nextStart.removeAll(nodes);
// store the edges in our results
if (detail == GRAPH_DETAIL_ALL)
{
links.addAll(edges);
}
else
{
for (Edge e : edges)
{
Vertex vOut = e.outVertex();
Vertex vIn = e.inVertex();
Vertex vTargetOut = computeParentForDetail(vOut, detail);
Vertex vTargetIn = computeParentForDetail(vIn, detail);
if (vTargetOut != null && vTargetIn != null && !vTargetOut.equals(vTargetIn))
{
// nodes.add(vTargetOut);
// nodes.add(vTargetIn);
virtualLinks.add(new CallGraphLink(nextLinkId.getAndIncrement(),
vTargetOut.id().toString(),
vTargetIn.id().toString(),
e.label()));
}
}
}
nodes.addAll(newNodes);
newNodes.clear();
// there is another level to gather, recurse, reducing the level parm value
level0.putAll(traverseLevel(nextStart,
direction,
levels - 1,
detail,
nodes,
links,
virtualLinks,
nextLinkId));
}
}
}
return level0;
}
/**
* Depending on the graph detail level, it will compute the parent vertex to be added to the
* emitted graph snippet.
*
* @param v
* The vertex.
* @param detail
* The graph detail level.
*
* @return If the detail is {@link #GRAPH_DETAIL_ALL}, return the vertex. Otherwise, one of
* its parents or {@code null} if the parent or vertex type doesn't match the detail
* level.
*/
private Vertex computeParentForDetail(Vertex v, int detail)
{
Vertex parent = null;
int resDomain = v.<Integer>property("resource-domain").value();
int nodeType = v.<Integer>property("node-type").value();
boolean external = v.<Boolean>property("external").isPresent() &&
v.<Boolean>property("external").value();
switch (detail)
{
case GRAPH_DETAIL_FILES:
if (resDomain == FILE_RESOURCE)
{
return v;
}
if (resDomain == AST_NODE)
{
parent = CallGraphHelper.findProcedureFile(v);
if (parent == null)
{
parent = CallGraphHelper.findSchemaFile(v);
}
return parent;
}
// check if this is a library procedure
if (nodeType == NATIVE_API)
{
return v.edges(Direction.IN, "contains").next().outVertex();
}
// what's left is an external target
return external || nodeType == INCLUDE_FILE ? v : null;
case GRAPH_DETAIL_ENTRY_POINTS:
if (resDomain == FILE_RESOURCE || nodeType == NATIVE_API)
{
return v;
}
if (resDomain == AST_NODE)
{
int[] parents =
{
FUNCTION,
INTERNAL_PROCEDURE,
METHOD_DEF,
DEFINE_PROPERTY_SET,
DEFINE_PROPERTY_GET,
CONSTRUCTOR,
DESTRUCTOR,
CLASS_DEF,
INTERFACE_DEF,
ENUM_DEF,
EXTERNAL_PROCEDURE,
PROCEDURE_FILE,
TABLE,
DATABASE,
SCHEMA_FILE
};
for (int ptype : parents)
{
parent = CallGraphHelper.findParentAstVertex(v, ptype);
if (parent != null)
{
break;
}
}
return parent;
}
return external ? v : null;
case GRAPH_DETAIL_ALL:
return v;
}
throw new IllegalArgumentException("Invalid graph detail level: " + detail);
}
/**
* Render the given sets of nodes and links as a call graph snippet. Since this is
* the last thing done with the graph database for the given API request, we commit
* the current transaction at the end of the rendering work.
*
* @param nextLinkId
* The counter which generates link IDs.
* @param nodes
* The vertices to include.
* @param links
* The edges to include.
* @param moreNodes
* The set in which to gather the artificial placeholder vertices.
* @param moreLinks
* The set in which to gather the artificial placeholder edges.
* @param rootList
* The list of root program files used to build the graph snippet.
*
* @return The snippet.
*/
private CallGraphSnippet createSnippet(AtomicInteger nextLinkId,
Set<Vertex> nodes,
Set<Edge> links,
Set<CallGraphNode> moreNodes,
Set<CallGraphLink> moreLinks,
String[] rootList)
{
List<CallGraphNode> nodeList = new ArrayList<>();
List<CallGraphLink> linkList = new ArrayList<>();
Map<String, CallSite> astDetails = new HashMap<>();
Set<String> nodeIds = new HashSet<>();
for (Vertex v : nodes)
{
nodeIds.add(v.id().toString());
}
for (CallGraphNode n : moreNodes)
{
nodeIds.add(n.getId());
}
for (Edge e : links)
{
if (!nodeIds.contains(e.outVertex().id().toString()))
{
// something was wrong, just add the vertex, for now....
nodes.add(e.outVertex());
nodeIds.add(e.outVertex().id().toString());
}
if (!nodeIds.contains(e.inVertex().id().toString()))
{
// something was wrong, just add the vertex, for now....
nodes.add(e.inVertex());
nodeIds.add(e.inVertex().id().toString());
}
linkList.add(new CallGraphLink(nextLinkId.getAndIncrement(),
e.outVertex().id().toString(),
e.inVertex().id().toString(),
e.label()));
}
for (Vertex v : nodes)
{
int nodeType = v.<Integer>property("node-type").value();
String label = describeNode(v, nodeType);
String vid = v.id().toString();
String tooltip = describeTooltip(v, nodeType);
nodeList.add(new CallGraphNode(vid, nodeType, label, tooltip));
CallSite astDet = buildCallSite(v);
if (astDet != null)
{
astDetails.put(vid, astDet);
}
}
nodeList.addAll(moreNodes);
moreLinks.forEach(l -> { if (!l.getSource().equals(l.getTarget())) linkList.add(l); } );
Collections.sort(linkList, new Comparator<CallGraphLink>()
{
@Override
public int compare(CallGraphLink o1, CallGraphLink o2)
{
return o1.getId() - o2.getId();
}
});
db.tx().commit();
return new CallGraphSnippet(nodeList, linkList, astDetails, rootList);
}
}