CallGraphWorker.java
/*
** Module : CallGraphWorker.java
** Abstract : provides a pattern engine service to create a new call graph based on matches
** found during AST processing
**
** Copyright (c) 2005-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ----------------------------Description-----------------------------
** 001 GES 20050311 @20310 Created initial version that provides support for creating the call
** graph in memory during pattern engine processing. It provides a
** dynamic iterator that walks the call graph and returns File objects
** for each node until the entire graph is complete. This iterator
** changes its return value dynamically based on inline changes to the
** graph. This is the key to driving the pattern engine from a single
** root entry point and being able to create an entire graph of any
** number of ASTs since the next AST for the pattern engine is read
** from the call graph that was just created by pattern engine
** processing.
** 002 GES 20050314 @20321 Added support for RUN VALUE() based call graph nodes using hints.
** 003 GES 20050316 @20397 Converted this worker to use the visitAst() interface in
** PatternWorker instead of using a rule driven approach.
** 004 ECF 20050526 @21290 Changes to support new expression engine implementation. Changed
** library registration mechanism based on superclass' modifications
** to support single library per pattern worker limit.
** 005 SIY 20050531 @21341 Minimal changes to enable CallGraphWorker use .ast files instead
** of .p.
** 006 SIY 20050606 @21453 Removed redundant methods for the call graph processing. Added
** methods for the "dead" files processing. Minor cleanups.
** 007 SIY 20050622 @21544 Added absolute application path handling.
** 008 SIY 20051011 @23018 Fixed null pointer exception during loading of the hints file.
** 009 GES 20060129 @24131 Minor name change for FileList.
** 010 GES 20060130 @24140 Safety code added to ensure that this worker could be used > 1 time
** per JVM process.
** 011 GES 20060130 @24157 Honor debug settings before writing output to the terminal.
** 012 SIY 20060312 @24981 Improved include files path handling to avoid incorrect marking them
** as not referenced.
** 013 GES 20090429 @42061 Match package and class name changes.
** 014 GES 20090514 @42233 Removed dead code.
** 015 CA 20140313 Moved to a graph DB implementation. Added support for various
** call-sites, mos
** 016 CA 20140408 Fixed filename matching for source-code originating from a different
** OS than the one where conversion is ran.
** 017 CA 20140424 Fixed disambiguation of call sites for which the hints provide no
** targets.
** 018 ECF 20160407 Added TODO in loadIncludeFiles to check hints for an include-spec
** setting.
** 019 OM 20170123 Removed dead code.
** 020 GES 20170424 Changes for call graph analysis v3.
** 021 CA 20180327 Added search index support for reverse-filename.
** Added function-name and libname indexes.
** Other performance optimizations.
** Reworked hints to use values like ext-prog.p:proc-name, instead of
** hint suffix.
** 022 CA 20180412 More indexes to improve performance.
** CA 20180413 nextExternalId is expensive, cache the current value in the graph
** and use that to compute next IDs.
** Cache the vertex parents in a parent-node-type-<node-type> property
** to avoid expensive computation.
** CA 20180414 Cache the non-virtual function/procedure vertices, as the application
** may have millions of these (including virtual), and executing millions
** of queries for each call-site or virtual entry point will be slow.
** CA 20180418 Mark all reachable entry points, to improve the dead code report.
** More indexes and other performance optimizations.
** CA 20180507 Compute line/col from first descendant with line/col different than 0.
** In update mode, re-process only files which have been processed.
** Added path aliasing (r-code to source) support.
** CA 20180509 Allow include-level hints to disambiguate call sites.
** CA 20180525 Callgraph improvements - changes in tx management, vertex caching
** and more.
** 023 CA 20181003 Added indexes for OO blocks and entry-point calculation.
** 024 CA 20181123 Basic support for multiple classes with the same qualified names
** CA 20181128 Fixed ambiguous callsite processing (getCallSiteHintTargets must
** return null if no hints are defined).
** 025 CA 20200428 Some changes to process DEFINE ENUM. Untested.
** 026 CA 20211214 The AST manager plugin must be context-local, for runtime conversion, as the
** InMemoryRegistryPlugin is not thread-safe.
** 027 TJD 20240208 Java 17 dependencies updates
** TJD 20240216 Newer versions of JanusDB require MixedTypeIndexes when using aggregates
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.uast;
import java.io.*;
import java.text.*;
import java.util.*;
import java.util.concurrent.atomic.*;
import java.util.function.*;
import com.goldencode.ast.*;
import com.goldencode.graphdb.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.pattern.*;
import com.goldencode.p2j.preproc.*;
import com.goldencode.p2j.util.*;
import org.janusgraph.graphdb.types.system.SystemTypeManager;
import org.apache.tinkerpop.gremlin.structure.*;
import org.apache.tinkerpop.gremlin.process.traversal.*;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.*;
import static org.janusgraph.core.attribute.Text.*;
/**
* Provides a pattern engine service to create a new call graph based on matches found during AST
* processing. Only one call graph can be created at a time due to use of static members.
* <p>
* This call graph contains external file references, schema tables and external link sites (as
* native processes or web services) AND all duplications have been removed.
* <p>
* This worker provides generic routines, so other call-sites can be added in the future.
* <p>
* Ambiguous call-sites can be disambiguated using UAST hints to specify 1 or more targets (see
* {@link UastHints}); note that such hints are named using the call-site-key as prefix,
* then an instance number specifying the 0-based index of this match in the given source file.
*/
public class CallGraphWorker
extends AbstractPatternWorker
implements ProgressParserTokenTypes
{
/**
* List of targets to be processed. This starts with the initial entry-points and acts
* as a queue to which called external programs are added.
*/
private static final LinkedList<Vertex> targets = new LinkedList<>();
/** The connected graph DB. */
private static Graph db = null;
/** Stores the current AST's hints object. */
private UastHints hints = null;
/** Caches the include-level hint files. */
private Map<String, UastHints> includeHints = new HashMap<>();
/** For each include file, keep a counter of hint IDs, for each hint type. */
private Map<String, Map<Integer, AtomicInteger>> includeHintsNextIds = new HashMap<>();
/** Stores the current AST's preprocessor hints object. */
private PreprocessorHints pphints = null;
/** <code>true</code> if no output should be written to the terminal. */
private boolean silent = false;
/** Flag indicating if we are starting to collect targets. */
private boolean collectTargets = false;
/** Map of aliases (r-code folders) to their source folders (conversion folders). */
private Map<String, String> pathAliases = null;
/**
* Default constructor which calls the super-class constructor, registers
* its libraries and initializes its instance members.
*
* @throws AstException
* Forwarded from underlying methods.
*/
public CallGraphWorker()
throws AstException
{
super();
setLibrary(new CallGraph());
// honor debug settings
silent = (PatternEngine.getDebugLevel() == PatternEngine.MSG_NONE);
// initialize the graph DB
Graph db = getGraph();
if (db.tx().isOpen())
{
db.tx().commit();
db.tx().close();
}
db.tx().open();
// load the path aliases
pathAliases = Configuration.getPathAliases();
}
/**
* Initialize the {@link #db graph DB} as necessary and return it.
* <p>
* The graph DB will connect to the folder specified via the {@code callgraph-db-folder}
* parameter and it will default to {@link CallGraphHelper#DEFAULT_CALLGRAPH_DB_FOLDER},
* if not set.
* <p>
* The graph DB is configured for a JanusGraph implementation, backed by {@code persistit}
* storage and with {@code lucene} indexes.
*
* @return The graph DB instance associated with the call-graph.
*/
public static Graph getGraph()
{
if (db == null)
{
db = CallGraphHelper.getGraph();
// unique indexes
GraphDB.createIndex(db, "os-filename", String.class, true, true);
GraphDB.createIndex(db, "filename", String.class, true, true);
GraphDB.createIndex(db, "reverse-filename", String.class, true, false, true);
GraphDB.createIndex(db, "reverse-compiled-filename", String.class, true, false, true);
GraphDB.createIndex(db, "port-type", String.class, true, true);
GraphDB.createIndex(db, "schemaname", String.class, true, true);
GraphDB.createIndex(db, "command", String.class, true, true);
GraphDB.createIndex(db, "connect-string", String.class, true, true);
GraphDB.createIndex(db, "com-target", String.class, true, true);
GraphDB.createIndex(db, "dde-target", String.class, true, true);
GraphDB.createIndex(db, "event-name", String.class, true, false);
GraphDB.createIndex(db, "procedure", String.class, true, false);
GraphDB.createIndex(db, "function", String.class, true, false);
GraphDB.createIndex(db, "missing_function", String.class, true, true);
GraphDB.createIndex(db, "missing_procedure", String.class, true, true);
GraphDB.createIndex(db, "class-name", String.class, true, false);
GraphDB.createIndex(db, "method-name", String.class, true, false);
GraphDB.createIndex(db, "property-name", String.class, true, false);
GraphDB.createIndex(db, "node-key", String.class, true, false);
GraphDB.createIndex(db, "function-name", String.class, true, false);
GraphDB.createIndex(db, "libname", String.class, true, false);
GraphDB.createIndex(db, "node-type", Integer.class, true, false, true);
GraphDB.createIndex(db, "node-id", Long.class, true, false, true);
GraphDB.createIndex(db, "call-site-key", Integer.class, false, false);
GraphDB.createIndex(db, "call-site-id", Long.class, false, false);
GraphDB.createIndex(db, "trigger-type", Integer.class, false, false);
GraphDB.createIndex(db, "live", Boolean.class, true, false);
GraphDB.createIndex(db, "reachable", Boolean.class, true, false);
GraphDB.createIndex(db, "external", Boolean.class, true, false);
GraphDB.createIndex(db, "included", Boolean.class, true, false);
GraphDB.createIndex(db, "resource-domain", Integer.class, true, false);
GraphDB.createIndex(db,
new String[] { "node-id", "node-type" },
new Class<?>[] { Long.class, Integer.class },
true, true);
GraphDB.createIndex(db,
new String[] { "node-id", "resource-domain" },
new Class<?>[] { Long.class, Integer.class },
true, false);
// GraphDB.createIndex(db,
// new String[] { "node-key", "external" },
// new Class<?>[] { String.class, Boolean.class },
// true, false);
GraphDB.createIndex(db,
new String[] { "function", "node-type" },
new Class<?>[] { String.class, Integer.class },
true, false);
GraphDB.createIndex(db,
new String[] { "procedure", "node-type" },
new Class<?>[] { String.class, Integer.class },
true, false);
GraphDB.createIndex(db,
new String[] { "function", "live" },
new Class<?>[] { String.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "procedure", "live" },
new Class<?>[] { String.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "method-name", "node-type" },
new Class<?>[] { String.class, Integer.class },
true, false);
GraphDB.createIndex(db,
new String[] { "node-type", "event-name" },
new Class<?>[] { Integer.class , String.class },
true, false);
GraphDB.createIndex(db,
new String[] { "resource-domain", "os-filename" },
new Class<?>[] { Integer.class, String.class },
true, false);
GraphDB.createIndex(db,
new String[] { "node-type", "app-entry-point" },
new Class<?>[] { Integer.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "external", "missing" },
new Class<?>[] { Boolean.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "procedure", "virtual" },
new Class<?>[] { String.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "function", "virtual" },
new Class<?>[] { String.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "node-type", "resource-domain", "reachable" },
new Class<?>[] { Integer.class, Integer.class, Boolean.class },
true, false);
GraphDB.createIndex(db,
new String[] { "external", "missing", "reachable" },
new Class<?>[] { Boolean.class, Boolean.class, Boolean.class },
true, false);
String[] extKeys =
{
"filename",
"port-type",
"command",
"connect-string",
"com-target",
"dde-target",
"procedure",
"function",
"missing_function",
"missing_procedure",
"class-name",
"method-name",
"property-name",
"function-name",
"libname",
};
for (String key : extKeys)
{
GraphDB.createIndex(db,
new String[] { key, "node-key", "external" },
new Class<?>[] { String.class, String.class, Boolean.class },
true, false);
}
int[] parents =
{
PROCEDURE_FILE,
SCHEMA_FILE,
FUNCTION,
INTERNAL_PROCEDURE,
EXTERNAL_PROCEDURE,
TABLE,
DATABASE,
TRIGGER_BLOCK,
METHOD_DEF,
DEFINE_PROPERTY_SET,
DEFINE_PROPERTY_GET,
CONSTRUCTOR,
DESTRUCTOR,
CLASS_DEF,
INTERFACE_DEF,
ENUM_DEF
};
for (int p : parents)
{
String prop = CallGraphHelper.parentProperty(p);
GraphDB.createIndex(db, prop, Long.class, true, false);
GraphDB.createIndex(db,
new String[] { prop, "entry-point" },
new Class<?>[] { String.class, Boolean.class },
true, false);
}
String prop = CallGraphHelper.parentProperty(PROCEDURE_FILE);
GraphDB.createIndex(db,
new String[] { "node-type", prop },
new Class<?>[] { Integer.class , Long.class },
true, false);
}
return db;
}
/**
* Creates a new iterator that uses as initial nodes the explicit list of programs passed at
* rootNodes parameter and also all the external programs with ambiguous call-sites.
* <p>
* The explicit targets in the rootNodes array will not be marked as application entry-points.
* <p>
* The call graph is walked dynamically, using the nodes as the tree is being built to properly
* direct the pattern engine in its processing, and avoiding already-processed nodes.
*
* @param rootNodes
* List of root AST names to use as entry points.
* @param relative
* Specifies if the name is relative to the project home directory. If
* <code>false</code>, the name is either an absolute filename or it is relative to
* the current directory.
*
* @return <code>Iterator</code> instance suitable for the node list traversal.
*/
static Iterator<Vertex> newUpdateCallGraphIterator(String[] rootNodes, boolean relative)
{
long aid = CallGraphHelper.AMBIGUOUS_NODE_ID;
targets.clear();
loadRootNodes(rootNodes, relative, false);
// ensure the AST Manager is initialized
new PatternEngine();
AstManager mgr = AstManager.get();
// load the ambiguous nodes
Vertex ambiguous = CallGraphHelper.findNodeById(AMBIGUOUS, aid);
Iterator<Vertex> itr = db.traversal().V(ambiguous).in().has("reachable", true);
Set<String> ambigExtProgs = new HashSet<>();
while (itr.hasNext())
{
Vertex node = itr.next();
String filename = mgr.getTreeName((long) node.<Long>property("node-id").value());
if (filename != null)
{
filename = CallGraphHelper.prepareFilename(filename);
Vertex procFile = CallGraphHelper.findUniqueNode("filename", filename);
if (!ambigExtProgs.contains(filename) && procFile.property("finished").isPresent())
{
ambigExtProgs.add(filename);
procFile.<Boolean>property("processing", true);
targets.add(procFile);
}
}
}
return targetsIterator();
}
/**
* Creates a new iterator that walks a call graph dynamically, using the nodes as the tree is
* being built to properly direct the pattern engine in its processing. Advancing to the next
* node is done when the tree is changed rather than when the next() method is called.
* <p>
* The explicit targets in the rootNodes array will be marked as application entry-points.
*
* @param rootNodes
* List of root AST names to use as entry points.
* @param relative
* Specifies if the name is relative to the project home directory. If
* <code>false</code>, the name is either an absolute filename or it is relative to
* the current directory.
*
* @return <code>Iterator</code> instance suitable for the node list traversal.
*/
static Iterator<Vertex> newCallGraphIterator(String[] rootNodes, boolean relative)
{
// clear the targets and load the root nodes
targets.clear();
loadRootNodes(rootNodes, relative, true);
return targetsIterator();
}
/**
* Create a new iterator over the {@link #targets} queue, by polling elements from the
* head of the queue until the queue is empty.
*
* @return See above.
*/
private static Iterator<Vertex> targetsIterator()
{
return new Iterator<Vertex>()
{
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
@Override
public Vertex next()
{
return targets.pollFirst();
}
@Override
public boolean hasNext()
{
return !targets.isEmpty();
}
};
}
/**
* Load the specified root nodes into the {@link #targets}.
*
* @param rootNodes
* List of root AST names to use as entry points.
* @param relative
* Specifies if the name is relative to the project home directory. If
* <code>false</code>, the name is either an absolute filename or it is relative to
* the current directory.
* @param entryPoint
* Flag to mark or not the nodes as application entry-points.
*/
private static void loadRootNodes(String[] rootNodes,
boolean relative,
boolean entryPoint)
{
if (rootNodes == null)
{
return;
}
List<Vertex> nodes = CallGraphHelper.loadFileNodes(rootNodes, relative);
for (Vertex v : nodes)
{
if (entryPoint)
{
v.<Boolean>property("app-entry-point", true);
}
v.<Boolean>property("processing", true);
targets.add(v);
}
}
/**
* Reverse the given string.
*
* @param s
* The string to reverse.
*
* @return The reversed string.
*/
private static String reverseString(String s)
{
char[] chars = s.toCharArray();
for (int i = 0; i < chars.length / 2; i++)
{
// switch chars in-place
int idx2 = chars.length - (i + 1);
char aux = chars[i];
chars[i] = chars[idx2];
chars[idx2] = aux;
}
return new String(chars);
}
/**
* Resets the current hints object and the counters for all call-sites and each time a new AST
* is visited. This enables the lazy loading of hints files only when needed and provides the
* ability to calculate the hint name corresponding to a specific call-site in a given file.
*
* @param ast
* The root node of the source AST about to be processed by the pattern engine.
*/
public void visitAst(Aast ast)
{
hints = null;
pphints = null;
includeHintsNextIds.clear();
}
/**
* A container for the hints referring to a call-site disambiguation.
*/
public class CallSiteHint
{
/** The hint's suffix. */
private final String suffix;
/** The call-site instance (numbered in order of appearance). */
private final int instance;
/** The index in the hints's array, otherwise -1. */
private final int index;
/** An additional info. */
private final int additional;
/** Flag indicating this is from an include. */
private boolean include;
/**
* Create a new call-site hint.
*
* @param instance
* The call-site instance (numbered in order of appearance).
* @param index
* The index in the hints's array, otherwise -1.
* @param additional
* An additional info.
* @param suffix
* The hint's suffix.
* @param include
* Flag indicating this hint is from an include file.
*/
public CallSiteHint(int instance, int index, int additional, String suffix, boolean include)
{
this.instance = instance;
this.index = index;
this.additional = additional;
this.suffix = suffix;
this.include = include;
}
/**
* Determine if this hint is from an include file or not.
*
* @return See above.
*/
public boolean isInclude()
{
return include;
}
/**
* Get the hint's suffix.
*
* @return The {@link #suffix}.
*/
public String getSuffix()
{
return suffix;
}
/**
* Create a string representation of this instance.
*
* @return See above.
*/
@Override
public String toString()
{
StringBuilder sb = new StringBuilder();
sb.append(instance);
if (index >= 0)
{
sb.append("_").append(index);
}
if (additional >= 0)
{
sb.append(" ").append(additional);
}
if (suffix != null && !suffix.isEmpty())
{
sb.append(" ").append(suffix);
}
return sb.toString();
}
/**
* Compute the hashcode of this instance.
*
* @return See above.
*/
@Override
public int hashCode()
{
int result = 17;
result = result * 37 + instance;
result = result * 37 + index;
result = result * 37 + additional;
result = result * 37 + (suffix == null ? 0 : suffix.hashCode());
return result;
}
/**
* Check if this instance is the same as the given one.
*
* @param obj
* The instance to compare against.
*
* @return See above.
*/
@Override
public boolean equals(Object obj)
{
if (!(obj instanceof CallSiteHint))
{
return false;
}
CallSiteHint csh = (CallSiteHint) obj;
return csh.instance == instance &&
csh.index == index &&
csh.additional == additional &&
((csh.suffix == null && suffix == null) || csh.suffix.equals(suffix));
}
}
/**
* Provides a service for creating call graph nodes in a tree starting at some root node.
* This class is a <code>PatternEngine</code> worker.
*/
public class CallGraph
{
/** The number of created edges since last opened transaction. */
private long txNewEdges = 0;
/** The number of created vertices since last opened transaction. */
private long txNewVertices = 0;
/** The number of set properties since last opened transaction. */
private long txChangedProperties = 0;
/** A cache of computed node IDs for external targets. */
private Map<Integer, AtomicLong> externalIds = new HashMap<>();
/**
* Map of non-virtual functions (by name with vertex ID list). There are 3 indices:
* <ul>
* <li>index 0 - only vertex IDs part of ADM programs.</li>
* <li>index 1 - only vertex IDs NOT part of ADM programs.</li>
* <li>index 2 - all vertex IDs from both 0 and 1 indexes.</li>
* </ul>
*/
private Map<String, List<Object>[]> functions = null;
/**
* Map of non-virtual procedures (by name with vertex ID list). There are 3 indices:
* <ul>
* <li>index 0 - only vertex IDs part of ADM programs.</li>
* <li>index 1 - only vertex IDs NOT part of ADM programs.</li>
* <li>index 2 - all vertex IDs from both 0 and 1 indexes.</li>
* </ul>
*/
private Map<String, List<Object>[]> procedures = null;
/** The ambiguous vertex. */
private Vertex ambiguousVertex = null;
/**
* Set the new state of the {@link CallGraphWorker#collectTargets} flag.
*
* @param state
* {@code true} to start collecting new targets, {@code false} otherwise.
*/
public void setCollectTargets(boolean state)
{
collectTargets = state;
}
/**
* Get the include source file for this AST. If the source is not from an include file,
* return {@code null}.
*
* @param ast
* The AST for which the include source is needed.
* @param line
* The line in the source code.
* @param column
* The column in the source code.
*
* @return See above.
*
* @throws PreprocessorException
* An unknown or unexpected failure.
*/
public String getIncludeSourceFor(Aast ast, int line, int column)
throws PreprocessorException
{
if (ast.isRoot())
{
return null;
}
if (line == 0 || column == 0)
{
final String msg = "WARNING: can not determine the include file for synthetic ASTs, " +
"with no line/column information: %d";
println(String.format(msg, ast.getId()));
return null;
}
PreprocessorHints hints = loadPreprocessorHints(ast);
String includeSource = hints.getSourceFor(line, column);
includeSource = CallGraphHelper.prepareFilename(includeSource);
String filename = CallGraphHelper.prepareFilename(ast.getFilename());
return (includeSource.equals(filename)) ? null : includeSource;
}
/**
* Prepare the procedure file name by replacing all windows-style separators with linux-style
* separators and making the name lowercase, if we are on a case-insensitive OS.
*
* @param filename
* Source file name.
*
* @return Prepared file name.
*/
public String prepareFilename(String filename)
{
return CallGraphHelper.prepareFilename(filename);
}
/**
* Mark this vertex and all its "contains" children as being live - this means the vertex
* was reached during call graph processing, and all its internal entries are available
* for targeting.
*
* @param v
* The procedure vertex to mark.
*/
public void markLive(Vertex v)
{
if (v.property("live").isPresent())
{
return;
}
property(v, "live", true);
traversalIterator(v, "contains", true).forEachRemaining(vp -> markReachable(vp));
}
/**
* Mark all reachable vertices from this node as 'reachable'.
*
* @param v
* The node from which to start reachable marking.
*/
public void markReachable(Vertex v)
{
if (v.property("reachable").isPresent())
{
return;
}
property(v, "reachable", true);
traversalIterator(v, "contains", false).forEachRemaining(vp -> markReachable(vp));
db.traversal().V(v)
.outE("references", "inherits", "implements", "override", "listens", "includes")
.inV()
.hasNot("reachable")
.forEachRemaining(ref -> markReachable(ref));
}
/**
* Check if we are in a case-sensitive OS, by reading the {@code case-sensitive} configuration
* parameter.
*
* @return {@code true} if the OS filenames are case-sensitive.
*/
public boolean isCaseSensitive()
{
return CallGraphHelper.isCaseSensitive();
}
/**
* Get the graph node to which all ambiguous call-sites link. This is temporary until the
* call-site ASTs are added into the callgraph.
*
* @return See above.
*/
public Vertex ambiguousNode()
{
if (ambiguousVertex != null)
{
return ambiguousVertex;
}
long aid = CallGraphHelper.AMBIGUOUS_NODE_ID;
Vertex node = CallGraphHelper.findNodeById(AMBIGUOUS, aid);
if (node == null)
{
node = createNode("ambiguous", aid, AMBIGUOUS, UNKNOWN_RESOURCE);
}
ambiguousVertex = node;
return node;
}
/**
* For the specified node, remove all the edges in the given direction which match the
* label, property key and property value.
*
* @param node
* A graph node.
* @param dir
* {@code true} for out edges and {@code false} for in edges.
* @param label
* The edge's label.
* @param key
* The property's key.
* @param value
* The property's value.
*/
public void removeEdges(Vertex node, boolean dir, String label, String key, Object value)
{
GraphTraversal<?, Vertex> gv = db.traversal().V(node);
GraphTraversal<?, Edge> ge = dir ? gv.outE() : gv.inE();
ge.hasLabel(label).has(key, value).forEachRemaining(edge -> edge.remove());
}
/**
* For the specified node, remove all the edges in the given direction which match the
* property key and property value.
*
* @param node
* A graph node.
* @param dir
* {@code true} for out edges and {@code false} for in edges.
* @param key
* The property's key.
* @param value
* The property's value.
*/
public void removeEdges(Vertex node, boolean dir, String key, Object value)
{
GraphTraversal<?, Vertex> gv = db.traversal().V(node);
GraphTraversal<?, Edge> ge = dir ? gv.outE() : gv.inE();
ge.has(key, value).forEachRemaining(edge -> edge.remove());
}
/**
* Create a new graph DB node associated with the given AST, representing a permanent schema
* table.
*
* @param schema
* The root database schema vertex.
* @param ast
* The schema table AST.
* @param schemaname
* The table's full name.
*
* @return The created {@link Vertex node}.
*
* @throws PreprocessorException
* An unknown or unexpected failure.
*/
public Vertex addTable(Vertex schema, Aast ast, String schemaname)
throws PreprocessorException
{
Vertex node = createAstNode(schema, ast, TABLE, true);
node.<String>property("schemaname", schemaname);
node.<String>property("node-key", "schemaname");
return node;
}
/**
* Remove the specified property from the given vertex or edge.
*
* @param element
* The vertex or edge from which to remove the property.
* @param key
* The property name.
*/
public void removeProperty(Element element, String key)
{
Property<?> vp = element.property(key);
if (vp.isPresent())
{
vp.remove();
++txChangedProperties;
commit(false);
}
}
/**
* Get the specified property from the given vertex or edge.
*
* @param element
* The vertex or edge on which to set the property.
* @param key
* The property name.
*
* @return The property's value or <code>null</code> if is not present at the node.
*/
public Object property(Element element, String key)
{
return CallGraphHelper.property(element, key);
}
/**
* Set the given property on the given vertex or edge. This is a helper function to handle
* the lack of generics support in TRPL.
*
* @param element
* The vertex or edge on which to set the property.
* @param key
* The property name.
* @param value
* The value to set.
*/
public void property(Element element, String key, Object value)
{
CallGraphHelper.property(element, key, value);
++txChangedProperties;
commit(false);
}
/**
* Create a new graph DB node associated with the given file resource.
*
* @param label
* The label for the new vertex.
* @param type
* The "node-type" property for the new vertex.
* @param filename
* The OS-level filename of the resource.
*
* @return The created {@link Vertex node}.
*/
public Vertex createFileResource(String label, int type, String filename)
{
long id = nextExternalId(type);
Vertex fnode = createNode(label, id, type, FILE_RESOURCE);
// need to use linux-style separators
String fullName = CallGraphHelper.prepareFilename(filename);
fnode.<String>property("os-filename", filename);
fnode.<String>property("filename", fullName);
fnode.<String>property("reverse-filename", reverseString(fullName));
fnode.<String>property("node-key", "filename");
if (type == PROCEDURE_FILE)
{
String compiledName = fullName;
if (!compiledName.toLowerCase().endsWith(".r"))
{
int lastDot = compiledName.lastIndexOf(".");
compiledName = compiledName.substring(0, lastDot) + ".r";
}
fnode.<String>property("reverse-compiled-filename", reverseString(compiledName));
}
return fnode;
}
/**
* Find or create the vertex associated with the specified event.
*
* @param container
* The container where the event is being listened.
* @param event
* The event name, kept at the <code>event-name</code> vertex property.
*
* @return See above.
*/
public Vertex findOrCreateEvent(Vertex container, String event)
{
if (event.length() == 0)
{
throw new IllegalStateException("Could not create an event with empty name!");
}
if (event.length() > 1)
{
event = event.toUpperCase();
}
Iterator<Vertex> iter = db.traversal()
.V(container)
.outE("contains")
.inV()
.has("node-type", EVENT)
.has("event-name", event)
.limit(1);
if (iter.hasNext())
{
return iter.next();
}
long id = nextExternalId(EVENT);
Vertex fnode = createNode(event, id, EVENT, EVENT);
fnode.<String>property("node-key", "event-name");
fnode.<String>property("event-name", event);
createEdge(container, fnode, "contains");
return fnode;
}
/**
* Create a new graph DB node associated with the given AST which is by some external
* resource. There will be a "contains" edge between the new node (in) and the
* containing node (out). The state of the AST will be available via properties.
*
* @param cntr
* The external resource (often a procedure file) that contains this node.
* @param ast
* The AST representing the entry point.
* @param type
* The "node-type" property for the new vertex.
* @param entry
* <code>true</code> if this is an entry-point, <code>false</code> for a
* call-site. This will be stored in a property called "entry-point".
*
* @return The created {@link Vertex node}.
*
* @throws PreprocessorException
* An unknown or unexpected failure.
*/
public Vertex createAstNode(Vertex cntr, Aast ast, int type, boolean entry)
throws PreprocessorException
{
// create the AST vertex
Vertex node = createNode(ast.getSymbolicTokenType(),
ast.getId(),
type,
AST_NODE);
// create the linkage to the procedure file in which this exists
Edge edge = createEdge(cntr, node, "contains");
// remember if it is a call-site or entry-point
node.<Boolean>property("entry-point", entry);
Aast ref = ast;
int line = ref.getLine();
int column = ref.getColumn();
while ((line == 0 || column == 0) && ref.getFirstChild() != null)
{
ref = (Aast) ref.getFirstChild();
line = ref.getLine();
column = ref.getColumn();
}
// save core AST state values as properties
node.<Integer>property("column", column);
node.<Integer>property("line", line);
node.<String>property("text", ast.getText());
node.<Integer>property("type", ast.getType());
node.<String>property("node-key", "ast-type");
node.<String>property("ast-type", ProgressParser.lookupTokenName(ast.getType()));
// load all annotations from this AST
Iterator<String> itr = ast.annotationKeys();
while (itr.hasNext())
{
String key = itr.next();
Object val = ast.getAnnotation(key);
if (node.property(key).isPresent() || SystemTypeManager.isSystemType(key))
{
if (!"type".equals(key))
{
println(String.format("WARNING: renaming key [%s] to [annotation_%s]", key, key));
}
key = "annotation_" + key;
}
// graph doesn't support list properties
if (!(val instanceof List))
{
node.<Object>property(key, val);
}
}
String includeSource = getIncludeSourceFor(ast, line, column);
boolean included = includeSource != null;
property(node, "included", included);
cacheParents(ast, node);
if (included)
{
Vertex include = CallGraphHelper.findUniqueNode("filename", includeSource);
// the direction must be from the 4GL code to the include file
createEdge(node, include, "includes");
// the original location in the include file can not be determined
if (!entry)
{
setIncludeHints(node, type);
}
}
return node;
}
/**
* Find an edge between the two nodes, which has the specified label and a property with
* the given key and value.
*
* @param out
* The source node.
* @param in
* The target node.
* @param label
* The edge's label.
* @param edgeKey
* The property's key.
* @param edgeValue
* The property's value.
*
* @return The first found edge or {@code null} if not found.
*/
public Edge findEdge(Vertex out, Vertex in, String label, String edgeKey, Object edgeValue)
{
Iterator<Edge> iter = db.traversal()
.V(out)
.outE(label)
.has(edgeKey, edgeValue)
.as("e")
.inV().hasId(in.id()).select("e");
return iter.hasNext() ? iter.next() : null;
}
/**
* Find an edge between the two nodes, which has the specified label.
*
* @param out
* The source node.
* @param in
* The target node.
* @param label
* The edge's label.
*
* @return The first found edge or {@code null} if not found.
*/
public Edge findEdge(Vertex out, Vertex in, String label)
{
Iterator<Edge> iter = db.traversal()
.V(out)
.outE(label)
.filter(e -> e.get().inVertex().equals(in));
return iter.hasNext() ? iter.next() : null;
}
/**
* Create an external node with the given type.
*
* @param label
* The label for the new vertex.
* @param type
* The "node-type" property for the new vertex.
* @param domain
* The token type of the node's resource domain, will be stored as the
* "resource-domain" property.
*
* @return The created {@link Vertex node}.
*/
public Vertex createExternalNode(String label, int type, int domain)
{
Vertex node = createNode(label, nextExternalId(type), type, domain);
node.<Boolean>property("external", true);
return node;
}
/**
* Find the external node with the specified property.
*
* @param nodeKey
* The property name.
* @param nodeValue
* The property value.
*
* @return The found node, but only if only one node was found.
*/
public Vertex findExternalNode(String nodeKey, Object nodeValue)
{
Iterator<Vertex> iter = db.traversal().V().has(nodeKey, nodeValue)
.has("node-key", nodeKey)
.has("external", true)
.limit(2);
Vertex v = null;
if (iter.hasNext())
{
v = iter.next();
if (iter.hasNext())
{
throw new IllegalStateException("More than one node found!");
}
}
return v;
}
/**
* Create a new {@link Vertex node} with the specified ID and type.
*
* @param label
* The label for the new vertex.
* @param id
* The node's unique identifier, will be stored as the "node-id" property.
* @param type
* The token type for the node, will be stored as the "node-type" property.
* @param domain
* The token type of the node's resource domain, will be stored as the
* "resource-domain" property.
*
* @return The created {@link Vertex node}.
*/
public Vertex createNode(String label, long id, int type, int domain)
{
if (label.trim().isEmpty())
{
// placeholder for SPACE - as Janus doesn't allow whitespace string
label = "SPACE";
}
Vertex node = db.addVertex(label);
node.<Long>property("node-id", id);
node.<Integer>property("node-type", type);
node.<Integer>property("resource-domain", domain);
++txNewVertices;
commit(false);
return node;
}
/**
* Create a new edge between the specified nodes, with the given label.
*
* @param out
* The source node.
* @param in
* The target node.
* @param label
* The edge label.
*
* @return The created {@link Edge}.
*/
public Edge createEdge(Vertex out, Vertex in, String label)
{
Edge edge = out.addEdge(label, in);
// this is useful only for the GraphML - as the edge's label is not serialized
edge.<String>property("edge-type", label);
++txNewEdges;
commit(false);
return edge;
}
/**
* Check if the two lists of comma-separated parameter data types match.
*
* @param expected
* The expected data types.
* @param actual
* The actual data types.
*
* @return <code>true</code> if the two match.
*/
public boolean parameterTypeMatch(String expected, String actual)
{
if (expected.isEmpty() && actual.isEmpty())
{
return true;
}
if (expected.isEmpty() != actual.isEmpty())
{
return false;
}
String[] stl1 = expected.split(",");
String[] stl2 = actual.split(",");
if (stl1.length != stl2.length)
{
return false;
}
for (int i = 0; i < stl1.length; i++)
{
String t1 = stl1[i].toLowerCase();
String t2 = stl2[i].toLowerCase();
if (t1.equals("poly") || t2.equals("poly"))
{
continue;
}
if (!ExpressionConversionWorker.isAssignmentCompatible(t1, t2))
{
// check if these are classes... if so, the expected must be the same or a super
// class of actual
Iterator<Vertex> ecls = CallGraphHelper.findClassNode(t1);
if (!ecls.hasNext())
{
return false;
}
Iterator<Vertex> acls = CallGraphHelper.findClassNode(t2);
if (!acls.hasNext())
{
return false;
}
Set<String> supers = classHierarchy(t2);
if (!supers.contains(t1))
{
return false;
}
}
}
return true;
}
/**
* Resolve the class hierarchy starting with the current class.
*
* @param clsName
* The class name.
*
* @return The inherits hierarchy from the given class.
*/
public Set<String> classHierarchy(String clsName)
{
clsName = clsName.toLowerCase();
Set<String> res = new LinkedHashSet<>();
res.add(clsName);
Iterator<Vertex> clsVIter = CallGraphHelper.findClassNode(clsName);
if (!clsVIter.hasNext())
{
System.out.println(String.format("WARNING: Class %s is not loaded in the callgraph!",
clsName));
return res;
}
while (clsVIter.hasNext())
{
Vertex clsV = clsVIter.next();
Iterator<Vertex> superC = db.traversal().V(clsV).outE("inherits").inV();
if (superC.hasNext())
{
res.addAll(classHierarchy(superC.next().property("class-name").value().toString()));
}
// add all implemented interfaces, sorted by their index
Iterator<Edge> superI = db.traversal().V(clsV).outE("implements");
List<Edge> esupers = new ArrayList<>();
while (superI.hasNext())
{
esupers.add(superI.next());
}
esupers.sort(new Comparator<Edge>()
{
@Override
public int compare(Edge e1, Edge e2)
{
int idx1 = (int) property(e1, "implements-index");
int idx2 = (int) property(e2, "implements-index");
return idx1 - idx2;
}
});
superI = esupers.iterator();
while (superI.hasNext())
{
Edge esuper = superI.next();
res.addAll(classHierarchy(esuper.inVertex().property("class-name").value().toString()));
}
if ((int) property(clsV, "node-type") == CLASS_DEF)
{
res.add(SymbolResolver.ROOT_OBJ_NAME);
}
}
return res;
}
/**
* Create an iterator for all the nodes linked with an outbound edge which has a property
* with the specified key and value.
*
* @param node
* The graph node.
* @param key
* The property key.
* @param value
* The property value.
*
* @return See above.
*/
public Iterator<Vertex> nodeIterator(Vertex node, String key, Object value)
{
return db.traversal().V(node).outE().has(key, value).inV();
}
/**
* Create an iterator for all the nodes linked with an outbound edge with the given label
* and the target node having the given property value.
*
* @param node
* The graph node.
* @param label
* The edge label.
* @param key
* The property key.
* @param value
* The property value.
* @param out
* The direction: out or in edges.
*
* @return See above.
*/
public Iterator<Vertex> nodeIterator(Vertex node,
String label,
String key,
Object value,
boolean out)
{
return out ? db.traversal().V(node).outE(label).inV().has(key, value)
: db.traversal().V(node).inE(label).outV().has(key, value);
}
/**
* Create an iterator for all the nodes linked with an outbound edge, which have a property
* with the specified key and value.
*
* @param node
* The graph node.
* @param key
* The property key.
* @param value
* The property value.
*
* @return See above.
*/
public Iterator<Vertex> nodeIteratorV(Vertex node, String key, Object value)
{
return db.traversal().V(node).outE().inV().has(key, value);
}
/**
* Create an iterator for all the nodes having a property with the specified key, regardless
* of value.
*
* @param key
* The property key.
*
* @return See above.
*/
public Iterator<Vertex> nodeIterator(String key)
{
return db.traversal().V().has(key);
}
/**
* Create a BFS node iterator starting with the given root, in the given direction.
*
* @param root
* The root node.
* @param dir
* The edge direction ({@code true} for out and {@code false} for in.
*
* @return See above.
*/
public Iterator<Vertex> traversalIterator(Vertex root, final boolean dir)
{
// start with our root
final LinkedList<Vertex> targets = new LinkedList<>();
targets.add(root);
final Set<Vertex> traversed = new HashSet<Vertex>();
traversed.add(root);
return new Iterator<Vertex>()
{
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
@Override
public Vertex next()
{
Vertex next = targets.removeFirst();
// collect all edges in the given direction
GraphTraversal<?, Vertex> gv = db.traversal().V(next);
Iterator<Vertex> iter = dir ? gv.outV() : gv.inV();
while (iter.hasNext())
{
Vertex node = iter.next();
if (!traversed.contains(node))
{
targets.addLast(node);
traversed.add(node);
}
}
return next;
}
@Override
public boolean hasNext()
{
return !targets.isEmpty();
}
};
}
/**
* Create a BFS node iterator starting with the given root.
*
* @param root
* The root node.
* @param label
* The labels for the edges which will be followed.
*
* @return See above.
*/
public Iterator<Vertex> traversalIterator(Vertex root, final String label)
{
return traversalIterator(root, label, true);
}
/**
* Create a BFS node iterator starting with the given root.
*
* @param root
* The root node.
* @param label
* The labels for the edges which will be followed.
* @param dir
* The edge direction ({@code true} for out and {@code false} for in.
*
* @return See above.
*/
public Iterator<Vertex> traversalIterator(Vertex root, final String label, boolean dir)
{
// start with our root
final LinkedList<Vertex> targets = new LinkedList<>();
targets.add(root);
final Set<Vertex> traversed = new HashSet<Vertex>();
traversed.add(root);
return new Iterator<Vertex>()
{
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
@Override
public Vertex next()
{
Vertex next = targets.removeFirst();
// collect all outbound edges with the given label
GraphTraversal<?, Vertex> gv = db.traversal().V(next);
Iterator<Vertex> iter = dir ? gv.outE(label).inV() : gv.inE(label).outV();
while (iter.hasNext())
{
Vertex node = iter.next();
if (!traversed.contains(node))
{
targets.addLast(node);
traversed.add(node);
}
}
return next;
}
@Override
public boolean hasNext()
{
return !targets.isEmpty();
}
};
}
/**
* Obtain a vertex traversal instance that starts at the given node.
*
* @param v
* The node at which to start the traversal.
*
* @return The traversal instance (which can also be used as an iterator).
*/
public GraphTraversal<?, Vertex> vertexTraversal(Vertex v)
{
return db.traversal().V(v);
}
/**
* Obtain a vertex traversal instance that starts at the given node, for IN edges.
*
* @param v
* The node at which to start the traversal.
*
* @return The traversal instance (which can also be used as an iterator).
*/
public Iterator<Edge> vertexTraversalInE(Vertex v)
{
return db.traversal().V(v).inE();
}
/**
* Create an iterator for all the nodes having a property with the specified key and value.
*
* @param key
* The property key.
* @param value
* The property value.
*
* @return See above.
*/
public Iterator<Vertex> nodeIterator(String key, Object value)
{
return db.traversal().V().has(key, value);
}
/**
* Create an iterator for all the nodes having a property with the specified key and value.
*
* @param key
* The property key.
* @param value
* The property value.
* @param key2
* The second property key.
* @param value2
* The second property value.
*
* @return See above.
*/
public Iterator<Vertex> nodeIterator(String key, Object value, String key2, Object value2)
{
return db.traversal().V().has(key, value).has(key2, value2);
}
/**
* Create an iterator for all the nodes having a property with the specified key and value,
* and without the specified property.
*
* @param key
* The property key.
* @param value
* The property value.
* @param without
* The property name which must not be present at the vertex.
*
* @return See above.
*/
public Iterator<Vertex> nodeIteratorWithout(String key, Object value, String without)
{
return db.traversal().V().has(key, value).hasNot(without);
}
/**
* Get all non-virtual functions with the specified name.
*
* @param name
* The function name.
* @param adm
* Flag indicating if we need functions from ADM programs or non-ADM programs.
*
* @return A vertex iterator.
*/
public Iterator<Vertex> functionsIterator(String name, boolean adm)
{
if (functions == null)
{
functions = loadInternalEntries(true);
}
List<Object>[] funcs = functions.get(name.toLowerCase());
return funcs == null ? Collections.emptyIterator() : vertexIterator(funcs[adm ? 0 : 1]);
}
/**
* Get all non-virtual functions with the specified name.
*
* @param name
* The function name.
*
* @return A vertex iterator.
*/
public Iterator<Vertex> functionsIterator(String name)
{
if (functions == null)
{
functions = loadInternalEntries(true);
}
List<Object>[] funcs = functions.get(name.toLowerCase());
return funcs == null ? Collections.emptyIterator() : vertexIterator(funcs[2]);
}
/**
* Get all non-virtual internal procedures with the specified name.
*
* @param name
* The procedure name.
* @param adm
* Flag indicating if we need functions from ADM programs or non-ADM programs.
*
* @return A vertex iterator.
*/
public Iterator<Vertex> proceduresIterator(String name, boolean adm)
{
if (procedures == null)
{
procedures = loadInternalEntries(false);
}
List<Object>[] procs = procedures.get(name.toLowerCase());
return procs == null ? Collections.emptyIterator() : vertexIterator(procs[adm ? 0 : 1]);
}
/**
* Get all non-virtual internal procedures with the specified name.
*
* @param name
* The procedure name.
*
* @return A vertex iterator.
*/
public Iterator<Vertex> proceduresIterator(String name)
{
if (procedures == null)
{
procedures = loadInternalEntries(false);
}
List<Object>[] procs = procedures.get(name.toLowerCase());
return procs == null ? Collections.emptyIterator() : vertexIterator(procs[2]);
}
/**
* Sort the given elements by the specified properties.
*
* @param elements
* A query result.
* @param props
* The property names, each element a {@link String}.
*
* @return The sorted elements.
*/
@SuppressWarnings("unchecked")
public Iterator<Element> sortedResults(Iterable<Element> elements, final Object[] props)
{
if (!elements.iterator().hasNext())
{
return Collections.EMPTY_LIST.iterator();
}
List<Element> sorted = new ArrayList<>();
for (Element e : elements)
{
sorted.add(e);
}
Collections.sort(sorted, new Comparator<Element>()
{
@Override
public int compare(Element o1, Element o2)
{
for (int i = 0; i < props.length; i++)
{
Object v1 = o1.<Object>property((String) props[i]).value();
Object v2 = o2.<Object>property((String) props[i]).value();
if (v1 != null && v1.equals(v2))
{
continue;
}
else if (v1 instanceof Comparable)
{
return ((Comparable<Object>) v1).compareTo(v2);
}
else
{
return 0;
}
}
return 0;
}
});
return sorted.iterator();
}
/**
* Create an iterator for all outbound edges with the given label.
*
* @param node
* The node for which the outbound edges are needed.
* @param label
* The edge's label.
*
* @return See above.
*/
public Iterator<Edge> edgeIterator(Vertex node, String label)
{
return db.traversal().V(node).outE(label);
}
/**
* Create an iterator for all outbound edges with the given label and property key and value.
*
* @param node
* The node for which the outbound edges are needed.
* @param label
* The edge's label.
* @param key
* Only edges with the having a property with this key.
* @param value
* Only edges with the having a property with this value.
*
* @return See above.
*/
public Iterator<Edge> edgeIterator(Vertex node, String label, String key, Object value)
{
return db.traversal().V(node).outE(label).has(key, value);
}
/**
* Create an iterator for all inbound edges with the given label.
*
* @param node
* The node for which the inbound edges are needed.
* @param label
* The edge's label.
*
* @return See above.
*/
public Iterator<Edge> edgeInIterator(Vertex node, String label)
{
return db.traversal().V(node).inE(label);
}
/**
* Find a node by its type and ID.
*
* @param nodeType
* The node's type.
* @param id
* The node's ID.
*
* @return The found node or {@code null} if it does not exist.
*/
public Vertex findNodeById(int nodeType, Object id)
{
return CallGraphHelper.findNodeById(nodeType, id);
}
/**
* Find the AST vertex in the {@link ProgressParserTokenTypes#AST_NODE} resource domain.
*
* @param id
* The node-id property value to match.
*
* @return The found vertex or <code>null</code>.
*/
public Vertex findAstNode(long id)
{
Iterator<Vertex> iter = db.traversal().V().has("node-id", id)
.has("resource-domain", AST_NODE)
.limit(1);
return iter.hasNext() ? iter.next() : null;
}
/**
* Find the adjacent vertex to the uniquely described vertex, via a unique edge
* matching the given label.
*
* @param type
* The "node-type" property.
* @param id
* The "node-id" property.
* @param label
* The label of the edge defining the relation between the two vertices.
* @param dir
* {@code true} for out edges and {@code false} for in edges.
*
* @return The found node or {@code null} if it does not exist.
*/
public Vertex findUniqueAdjacent(int type, Object id, String label, boolean dir)
{
Vertex node = CallGraphHelper.findNodeById(type, id);
Vertex result = null;
if (node != null)
{
GraphTraversal<?, Vertex> g = db.traversal().V(node);
Iterator<Vertex> iter = (dir ? g.out(label) : g.in(label)).limit(2);
while (iter.hasNext())
{
if (result == null)
{
result = iter.next();
}
else
{
throw new RuntimeException("More than 1 containing node found!");
}
}
}
return result;
}
/**
* Get the first adjacent node on the given direction, with the specified type and edge
* label.
*
* @param node
* The current node.
* @param type
* The found "node-type" property.
* @param label
* The label of the edge defining the relation between the two vertices.
* @param dir
* {@code true} for out edges and {@code false} for in edges.
*
* @return First found node or {@code null} if it does not exist.
*/
public Vertex findAdjacent(Vertex node, int type, String label, boolean dir)
{
return CallGraphHelper.findAdjacent(node, type, label, dir);
}
/**
* Get the first adjacent node on the given direction, with the specified type and edge
* label.
*
* @param node
* The current node.
* @param type
* The found "node-type" property.
* @param label
* The label of the edge defining the relation between the two vertices.
* @param prop
* The property name.
* @param val
* The property value.
* @param dir
* {@code true} for out edges and {@code false} for in edges.
*
* @return First found node or {@code null} if it does not exist.
*/
public Vertex findAdjacent(Vertex node,
int type,
String label,
String prop,
Object val,
boolean dir)
{
GraphTraversal<?, Vertex> g = db.traversal().V(node);
Iterator<Vertex> iter = dir ? g.out(label).has(prop, val).has("node-type", type).limit(1)
: g.in(label).has(prop, val).has("node-type", type).limit(1);
return (iter.hasNext() ? iter.next() : null);
}
/**
* Add a new target, to be processed.
* <p>
* Only {@code external-procedure} targets with no {@code finished} or {@code processing}
* annotations are added.
*
* @param node
* The node to be added.
*/
public void addTarget(Vertex node)
{
if (!collectTargets || node == null)
{
return;
}
Vertex vproc = null;
// add only AST targets and skip finished IDs and WIP IDs
if (node.<Integer>property("node-type").value().equals(EXTERNAL_PROCEDURE))
{
vproc = findUniqueAdjacent(EXTERNAL_PROCEDURE,
node.property("node-id").value(),
"contains",
false);
}
else if (node.<Integer>property("node-type").value().equals(PROCEDURE_FILE))
{
vproc = node;
}
if (vproc != null)
{
if (!vproc.<Boolean>property("finished").isPresent() &&
!vproc.<Boolean>property("processing").isPresent())
{
targets.addLast(vproc);
vproc.<Boolean>property("processing", true);
}
}
}
/**
* Commit the current transaction.
* <p>
* If the number of {@link #txNewVertices}, {@link #txNewEdges} plus {@link #txChangedProperties}
* is over 10000, then we force a commit.
*
* @param force
* Flag indicating if we force the commit.
*/
public void commit(boolean force)
{
if (!force && txNewVertices + txNewEdges + txChangedProperties > 10000)
{
force = true;
}
if (force)
{
// close and open the tx so that caches are evicted.
db.tx().commit();
db.tx().close();
db.tx().open();
txNewVertices = 0;
txNewEdges = 0;
txChangedProperties = 0;
}
}
/**
* Commit the current transaction.
*/
public void commit()
{
commit(true);
}
/**
* Return all class vertices with the qualified class name.
*
* @param className
* The qualified class name.
* @return An iterator with the found vertices.
*/
public Iterator<Vertex> findClassNode(String className)
{
return CallGraphHelper.findClassNode(className);
}
/**
* Find an unique node having a property with the given key and value.
*
* @param key
* The property key.
* @param val
* The property value.
*
* @return See above.
*
* @throws RuntimeException
* If more than one node is found.
*/
public Vertex findUniqueNode(String key, Object val)
{
return CallGraphHelper.findUniqueNode(key, val);
}
/**
* Find the parent vertex having the given type.
*
* @param node
* The node to find.
* @param type
* The parent's type.
*
* @return See above.
*/
public Vertex findParentAstVertex(Vertex node, int type)
{
return CallGraphHelper.findParentAstVertex(node, type);
}
/**
* Find the procedure file associated with the given AST node.
*
* @param node
* The node to find.
*
* @return See above.
*/
public Vertex findProcedureFile(Vertex node)
{
return CallGraphHelper.findProcedureFile(node);
}
/**
* Load all the include files (physical files) into the graph DB, as distinct nodes. This
* ensures that unreferenced/dead include files can be determined.
*/
public void loadIncludeFiles()
{
long start = System.currentTimeMillis();
String basepath = Configuration.getParameter("basepath", "./");
String spec = Configuration.getParameter("include-spec", "*.[iI]");
System.out.println(String.format("Using %s for include-file specifications.", spec));
FileSpecList list = new FileSpecList(new File(basepath), spec, true);
File[] includes = list.list();
for (File fi : includes)
{
String filename = fi.getPath();
System.out.println(filename);
Vertex vnode = CallGraphHelper.findUniqueNode("os-filename", filename);
if (vnode == null)
{
createIncludeFile(filename);
}
}
// commit the remainder transaction
commit();
long elapsed = System.currentTimeMillis() - start;
Date date = new Date(elapsed - TimeZone.getDefault().getRawOffset());
SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");
System.out.println("Done adding include-files in " + format.format(date));
}
/**
* Create the include graph for the given external program.
*
* @param ast
* The external program AST.
*
* @throws PreprocessorException
* An unknown or unexpected failure.
*/
public void createIncludeGraph(Aast ast)
throws PreprocessorException
{
PreprocessorHints ppHints = loadPreprocessorHints(ast);
HintsTreeNode root = ppHints.getRoot();
if (root != null)
{
// find the procedure file which links to each include file that is referenced
Vertex parent = findUniqueAdjacent(EXTERNAL_PROCEDURE, ast.getId(), "contains", false);
createIncludeGraph(parent.<Long>property("node-id").value(), null, root, parent);
}
}
/**
* Get the targets for a specific call-site (which is determined to be ambiguous),
* from the hints file.
*
* @param callSite
* The call site.
* @param callSiteKey
* The key of this call-site.
* @param instance
* The instance number for this call-site.
*
* @return A map of targets, where the key represents the hint no (and the index in the
* string array) for the target.
*/
public Map<CallSiteHint, String> getCallSiteHintTargets(Vertex callSite,
int callSiteKey,
int instance)
{
String lastUsedHints = (String) property(callSite, "last-used-hints");
Map<CallSiteHint, String> includeHints =
getCallSiteIncludeHintTargets(callSite, callSiteKey);
Map<CallSiteHint, String> res = getCallSiteHintTargets(hints,
callSiteKey,
instance,
false);
// if hints haven't changed, do nothing
String currentHints = null;
if (res == null)
{
currentHints = includeHints == null ? null : includeHints.toString();
}
else
{
currentHints = res.toString();
}
if (lastUsedHints == null && currentHints == null)
{
return null;
}
if (lastUsedHints != null && lastUsedHints.equals(currentHints))
{
return Collections.emptyMap();
}
if (res != null && includeHints != null)
{
// delete all previously 'include hints' edges
for (CallSiteHint csh : includeHints.keySet())
{
db.traversal().V(callSite)
.outE()
.has("included-hint", true)
.has("hint-id", csh.toString())
.forEachRemaining(e -> e.remove());
}
}
res = res != null ? res : includeHints;
if (res != null)
{
property(callSite, "last-used-hints", res.toString());
}
else if (lastUsedHints != null)
{
removeProperty(callSite, "last-used-hints");
}
return res;
}
/**
* Resolve all the external programs which match the given file name.
*
* @param name
* The filename.
*
* @return See above.
*/
public List<String> resolveExternalPrograms(String name)
{
Set<String> res = new TreeSet<>();
boolean hasAliases = pathAliases != null;
boolean caseSens = isCaseSensitive();
String filename = CallGraphHelper.prepareFilename(name);
java.util.function.Function<String, Boolean> isRcode =
(pname -> pname.endsWith(".r") || (!caseSens && pname.endsWith(".R")));
boolean legacyRcode = isRcode.apply(filename);
// TODO: RUN can target r-code even if extension is missing; i.e. RUN foo. will execute
// foo.r, if it exists
Consumer<String> lookupDirectName;
Consumer<String> lookupCompiledRcode;
Consumer<String> lookupAliasedDirectName;
Consumer<String> lookupAliasedCompiledRcode;
// we can't search in the propath and maybe resolve an unique name... the runtime propath
// is unknown at this time, thus resolve all matches
// instead of looking for a suffix, use the "reverse-filename" property to look for a
// prefix, which is indexed properly by then graph DB.
lookupDirectName = pname ->
{
// append a "/" prefix so that the name will match in any folder
String search = "/" + pname;
search = reverseString(search);
db.traversal().V().has("reverse-filename", textPrefix(search)).forEachRemaining(v ->
{
res.add((String) v.<String>property("filename").value());
});
};
lookupCompiledRcode = !legacyRcode ? null : (pname ->
{
if (!isRcode.apply(pname))
{
return;
}
// append a "/" prefix so that the name will match in any folder
String search = "/" + pname;
search = reverseString(search);
db.traversal().V()
.has("reverse-compiled-filename", textPrefix(search))
.forEachRemaining(v ->
{
res.add((String) v.<String>property("filename").value());
});
});
lookupAliasedDirectName = !hasAliases ? null : (pname ->
{
String[] aliases = SourceNameMapper.resolvePathAliases(pathAliases, pname);
for (String aliasName : aliases)
{
lookupDirectName.accept(aliasName);
}
});
lookupAliasedCompiledRcode = !legacyRcode || !hasAliases ? null : (pname ->
{
if (!isRcode.apply(pname))
{
return;
}
String[] aliases = SourceNameMapper.resolvePathAliases(pathAliases, pname);
for (String aliasName : aliases)
{
lookupCompiledRcode.accept(aliasName);
}
});
@SuppressWarnings("unchecked")
Consumer<String>[] lookups = new Consumer[]
{
lookupDirectName,
lookupCompiledRcode,
lookupAliasedDirectName,
lookupAliasedCompiledRcode,
};
for (Consumer<String> lookup : lookups)
{
if (lookup == null)
{
continue;
}
lookup.accept(filename);
}
if (res.isEmpty())
{
// if not found, add it, as it will be marked as missing
res.add(name);
}
return new ArrayList<>(res);
}
/**
* Print output to the terminal if debug settings allow.
*
* @param text
* Text to print.
*/
public void println(String text)
{
if (!silent)
{
System.out.println(text);
}
}
/**
* Loads the hints file for an AST if it exists and hasn't already been loaded.
*
* @return <code>true</code> if there are hints for this file and the hints are loaded,
* <code>false</code> if no hints are available.
*/
public boolean loadHints()
{
if (hints == null)
{
try
{
Aast src = getResolver().getSourceAst();
File file = Configuration.toFile(src.getFilename());
hints = new UastHints(file.getAbsolutePath());
}
catch (Exception exc)
{
hints = null;
// any failure drops us out
return false;
}
}
return true;
}
/**
* Create an include file vertex for the given filename. This will also check if there is
* an external program file already parsed, with the same filename. If so, it will create
* a program file, not an include file.
*
* @param filename
* The name of the file resource.
*/
public Vertex createIncludeFile(String filename)
{
try
{
AstManager.get().loadTree(filename + ".ast");
// this is a program file
Vertex vProg = findUniqueNode("os-filename", filename);
if (vProg == null)
{
vProg = createFileResource("Procedure File", PROCEDURE_FILE, filename);
}
return vProg;
}
catch (Exception e)
{
// ignore this
}
// it is a real include file
return createFileResource("Include File", INCLUDE_FILE, filename);
}
/**
* Process this call site and compute the details for defining include-level hints.
*
* @param callSite
* The call site.
* @param callSiteKey
* The key of this call-site.
*/
public void setIncludeHints(Vertex callSite, int callSiteKey)
{
if (!callSite.<Boolean>property("included").value() ||
callSite.property("include-hint-file").isPresent())
{
return;
}
// find the include file for this callSite
Iterator<Vertex> viter = db.traversal().V(callSite).outE("includes").inV();
Vertex includeRef = viter.hasNext() ? viter.next() : null;
if (includeRef == null)
{
return;
}
if (viter.hasNext())
{
// TODO: more than one include?
}
String includeFilename = includeRef.<String>property("filename").value();
Map<Integer, AtomicInteger> includeHintIds = includeHintsNextIds.get(includeFilename);
if (includeHintIds == null)
{
includeHintsNextIds.put(includeFilename, includeHintIds = new HashMap<>());
}
AtomicInteger nextId = includeHintIds.get(callSiteKey);
if (nextId == null)
{
includeHintIds.put(callSiteKey, nextId = new AtomicInteger(0));
}
int hintId = nextId.getAndIncrement();
property(callSite, "include-hint-file", includeFilename);
property(callSite, "include-hint-instance", hintId);
}
/**
* Load the include-level hints for the given include file.
*
* @param filename
* The include file name.
*
* @return The loaded hints or <code>null</code> if non exists.
*/
private UastHints loadIncludeHints(String filename)
{
UastHints hints = includeHints.get(filename);
if (hints != null)
{
return hints;
}
try
{
File file = Configuration.toFile(filename);
hints = new UastHints(file.getAbsolutePath());
}
catch (Exception exc)
{
return null;
}
includeHints.put(filename, hints);
return hints;
}
/**
* Load the associated include-level hints for this call site.
*
* @param callSite
* The call site.
* @param callSiteKey
* The key of this call-site.
*
* @return A map with the hints for this call site.
*/
private Map<CallSiteHint, String> getCallSiteIncludeHintTargets(Vertex callSite,
int callSiteKey)
{
if (!callSite.<Boolean>property("included").value())
{
return null;
}
int includeHintInstance = callSite.<Integer>property("include-hint-instance").value();
String includeHintFile = callSite.<String>property("include-hint-file").value();
UastHints includeHints = loadIncludeHints(includeHintFile);
if (includeHints == null)
{
return null;
}
return getCallSiteHintTargets(includeHints, callSiteKey, includeHintInstance, true);
}
/**
* Get the targets for a specific call-site (which is determined to be ambiguous),
* from the hints file.
*
* @param hints
* The hints to process.
* @param callSiteKey
* The key of this call-site.
* @param instance
* The instance number for this call-site.
* @param include
* Flag indicating this is from an include file.
*
* @return A map of targets, where the key represents the hint no (and the index in the
* string array) for the target.
*/
private Map<CallSiteHint, String> getCallSiteHintTargets(UastHints hints,
int callSiteKey,
int instance,
boolean include)
{
if (hints == null)
{
return null;
}
String suffixSep = ":";
StringBuilder sb = new StringBuilder(ProgressParser.lookupTokenName(callSiteKey));
sb.append("_");
sb.append(Integer.toString(instance));
String prefix = sb.toString();
String[] hintKeys = hints.getUastHints(prefix);
if (hintKeys == null || hintKeys.length == 0)
{
return null;
}
Map<CallSiteHint, String> res = new LinkedHashMap<>();
for (String key : hintKeys)
{
String cls = hints.getUastType(key);
if (String.class.getName().equals(cls))
{
String target = hints.getUastString(key);
String suffix = null;
int sufIdx = target.indexOf(suffixSep);
if (sufIdx > 0)
{
suffix = target.substring(sufIdx + 1);
target = target.substring(0, sufIdx);
if (suffix.isEmpty())
{
suffix = null;
}
}
List<String> extProgs = resolveExternalPrograms(target);
for (int j = 0; j < extProgs.size(); j++)
{
String extProg = extProgs.get(j);
if (extProgs.size() == 1)
{
res.put(new CallSiteHint(instance, -1, -1, suffix, include), target.trim());
}
else
{
res.put(new CallSiteHint(instance, -1, j, suffix, include), extProg);
}
}
}
else
{
// second try, maybe the hint is an array (we will get an exception if this isn't
// the case... this is OK)
String[] targets = hints.getUastStringArray(key);
// process each entry in the array
for (int i = 0; i < targets.length; i++)
{
String target = targets[i];
String suffix = null;
int sufIdx = target.indexOf(suffixSep);
if (sufIdx > 0)
{
suffix = target.substring(sufIdx + 1);
target = target.substring(0, sufIdx);
if (suffix.isEmpty())
{
suffix = null;
}
}
List<String> extProgs = resolveExternalPrograms(target);
for (int j = 0; j < extProgs.size(); j++)
{
String extProg = extProgs.get(j);
if (extProgs.size() == 1)
{
res.put(new CallSiteHint(instance, i, -1, suffix, include), extProg);
}
else
{
res.put(new CallSiteHint(instance, i, j, suffix, include), extProg);
}
}
}
}
}
return res;
}
/**
* Load all non-virtual internal entries.
*
* @param function
* Flag indicating if we want functions or internal procedures.
*
* @return A map of function/procedure names as keys and the associated IDs, for each
* name.
*/
private Map<String, List<Object>[]> loadInternalEntries(boolean function)
{
// 0 - ADM files, 1 - normal files
Map<String, List<Object>[]> res = new HashMap<>();
String prop = function ? "function" : "procedure";
Iterator<Vertex> iter = db.traversal().V().has("node-key", prop).hasNot("virtual");
while (iter.hasNext())
{
Vertex v = iter.next();
String name = (String) property(v, prop);
List<Object>[] list = res.get(name);
if (list == null)
{
res.put(name, list = new LinkedList[3]);
for (int i = 0; i < list.length; i++)
{
list[i] = new LinkedList<>();
}
}
Vertex vp = findProcedureFile(v);
List<Object> target = vp.property("adm-program").isPresent() ? list[0] : list[1];
target.add(v.id());
list[2].add(v.id());
}
return res;
}
/**
* Obtain a vertex iterator over the specified vertex IDs.
*
* @param vertexIds
* The vertex IDs.
*
* @return See above.
*/
private Iterator<Vertex> vertexIterator(List<Object> vertexIds)
{
if (vertexIds.isEmpty())
{
return Collections.emptyIterator();
}
Object[] vids = vertexIds.toArray();
return db.traversal().V(vids).iterate();
}
/**
* Cache the possible top-level block parents for this AST node.
* <p>
* The parent vertex ID is cached in the <code>parent-node-type-[vertex-id]</code> property
* at the vertex.
*
* @param ast
* The AST node.
* @param node
* The created vertex node.
*/
private void cacheParents(Aast ast, Vertex node)
{
boolean isSchema = ast.ancestor(-1, DATABASE) || ast.getType() == DATABASE;
int[] ancestors = null;
int nodeType = node.<Integer>property("node-type").value();
if (isSchema)
{
ancestors = new int[]
{
TABLE,
DATABASE
};
}
else
{
ancestors = new int[]
{
KW_ON,
TRIGGER_BLOCK,
FUNCTION,
INTERNAL_PROCEDURE,
METHOD_DEF,
DEFINE_PROPERTY_SET,
DEFINE_PROPERTY_GET,
CONSTRUCTOR,
DESTRUCTOR,
CLASS_DEF,
INTERFACE_DEF,
ENUM_DEF
};
}
boolean linkToExternal = true;
for (int ancestorType : ancestors)
{
if (ancestorType == nodeType)
{
linkToExternal = false;
continue;
}
int atype = ancestorType;
switch (ancestorType)
{
case KW_ON:
atype = KW_ON;
break;
case TRIGGER_BLOCK:
atype = TRIGGER_BLOCK;
break;
case FUNCTION:
atype = FUNCTION;
break;
case INTERNAL_PROCEDURE:
atype = PROCEDURE;
break;
case METHOD_DEF:
atype = METHOD_DEF;
break;
case DEFINE_PROPERTY_SET:
atype = DEFINE_PROPERTY;
break;
case DEFINE_PROPERTY_GET:
atype = DEFINE_PROPERTY;
break;
case CONSTRUCTOR:
atype = CONSTRUCTOR;
break;
case DESTRUCTOR:
atype = DESTRUCTOR;
break;
case CLASS_DEF:
atype = CLASS_DEF;
break;
case INTERFACE_DEF:
atype = INTERFACE_DEF;
break;
case ENUM_DEF:
atype = ENUM_DEF;
break;
}
Aast ancestor = ast.getAncestor(-1, atype);
if (ancestor == null)
{
continue;
}
switch (ancestorType)
{
case KW_ON:
// nothing to do
break;
case TRIGGER_BLOCK:
// nothing to do
break;
case FUNCTION:
ancestor = ancestor.getImmediateChild(KW_FUNCT, null);
break;
case INTERNAL_PROCEDURE:
ancestor = ancestor.getImmediateChild(KW_PROC, null);
break;
case METHOD_DEF:
ancestor = ancestor.getImmediateChild(KW_METHOD, null);
break;
case DEFINE_PROPERTY_SET:
// nothing to do
break;
case DEFINE_PROPERTY_GET:
// nothing to do
break;
case CONSTRUCTOR:
ancestor = ancestor.getImmediateChild(KW_CONSTRUC, null);
break;
case DESTRUCTOR:
ancestor = ancestor.getImmediateChild(KW_DESTRUCT, null);
break;
case CLASS_DEF:
ancestor = ancestor.getImmediateChild(KW_CLASS, null);
break;
case INTERFACE_DEF:
ancestor = ancestor.getImmediateChild(KW_INTERFAC, null);
break;
case ENUM_DEF:
ancestor = ancestor.getImmediateChild(KW_ENUM, null);
break;
}
Iterator<Vertex> iter = db.traversal().V()
.has("node-id", ancestor.getId())
.has("node-type", ancestorType)
.limit(1);
if (!iter.hasNext())
{
continue;
}
Vertex parent = iter.next();
if (parent != null)
{
String parentLabelName = CallGraphHelper.parentProperty(ancestorType);
property(node, parentLabelName, parent.id());
linkToExternal = false;
}
}
if (isSchema)
{
findParentAstVertex(node, SCHEMA_FILE);
}
else
{
if (linkToExternal)
{
// external procedure code is in its separate 'container', so use that if no
// other parent was found
Aast ancestor = ast.getAncestor(-1);
Vertex parent = db.traversal().V()
.has("node-id", ancestor.getId())
.has("node-type", EXTERNAL_PROCEDURE)
.limit(1)
.next();
if (parent != null)
{
String parentLabelName = CallGraphHelper.parentProperty(EXTERNAL_PROCEDURE);
property(node, parentLabelName, parent.id());
}
}
Vertex root = db.traversal().V()
.has("filename", CallGraphHelper.prepareFilename(ast.getFilename()))
.next();
String parentLabelName = CallGraphHelper.parentProperty(PROCEDURE_FILE);
property(node, parentLabelName, root.id());
}
}
/**
* Loads the preprocessor hints file for an AST if it exists and hasn't already been loaded.
*
* @return The loaded {@link PreprocessorHints} instance.
*
* @throws PreprocessorException
* An unknown or unexpected failure.
*/
private PreprocessorHints loadPreprocessorHints(Aast ast)
throws PreprocessorException
{
if (pphints == null)
{
pphints = new PreprocessorHints(ast.getFilename());
}
else
{
if (!ast.getFilename().equals(pphints.getSource()))
{
return new PreprocessorHints(ast.getFilename());
}
}
return pphints;
}
/**
* Create the include graph by walking the include tree specified by the root, recursively.
*
* @param fileSourceId
* The ID of the {@link ProgressParserTokenTypes#FILE_RESOURCE} vertex which
* started this include sub-graph.
* @param parentEdge
* The parent edge on which this include graph was created. May be {@code null}
* if the {@code root} hints is on depth 0.
* @param root
* The root include node.
* @param parent
* The parent node in the graph DB, to which the root's children will be attached.
*/
private void createIncludeGraph(long fileSourceId,
Edge parentEdge,
HintsTreeNode root,
Vertex parent)
{
@SuppressWarnings("unchecked")
Iterator<HintsTreeNode> iter = (Iterator<HintsTreeNode>) root.childIterator();
while (iter.hasNext())
{
HintsTreeNode node = iter.next();
IncludeHint hint = node.getIncludeHint();
String hintfile = hint.getFilename();
Vertex vnode = CallGraphHelper.findUniqueNode("os-filename", hintfile);
if (vnode == null)
{
final String msg = "WARNING: creating a new node for the include file %s, which " +
"was not found in initial include-file list.";
System.out.println(String.format(msg, hint.getFilename()));
// create it
vnode = createIncludeFile(hintfile);
}
Edge edge = createEdge(parent, vnode, "includes");
// copy the info: this needs to be set at the edge, as the same include file may
// appear multiple times
edge.<Integer>property("start-column", hint.getStartColumn());
edge.<Integer>property("end-column", hint.getEndColumn());
edge.<Integer>property("start-line", hint.getStartLine());
edge.<Integer>property("end-line", hint.getEndLine());
edge.<Integer>property("column", hint.getStartColumn());
edge.<Integer>property("line", hint.getStartLine());
edge.<Long>property("call-site-id", fileSourceId);
edge.<Integer>property("call-site-key", INCLUDES);
if (parentEdge != null)
{
edge.<String>property("parent-edge-id", parentEdge.id().toString());
}
createIncludeGraph(fileSourceId, edge, node, vnode);
}
}
/**
* Compute the next available ID for an external node. External nodes are targets which
* are not part of the code set ASTs (e.g. native processes, OCX controls, web service
* connections).
*
* @param type
* The external node type.
*
* @return The next available ID.
*/
private long nextExternalId(int type)
{
// when no vertex is found, the returned value is MIN_LONG - that's why we always
// increment
AtomicLong next = externalIds.get(type);
if (next != null)
{
return next.incrementAndGet();
}
// all external IDs are negative
GraphTraversal<Vertex, Comparable> maxNodeId = db.traversal()
.V()
.has("node-type", type)
.values("node-id")
.max();
Comparable<Long> result = null;
next = new AtomicLong();
if (maxNodeId.hasNext())
{
result = maxNodeId.next();
if (result != null && result instanceof Long)
{
next.set((Long) result + 1);
}
}
externalIds.put(type, next);
return next.get();
}
}
}