CallGraphGenerator.java
/*
** Module : CallGraphGenerator.java
** Abstract : drives the pattern engine service to create a call graph for one or more root
** entry points
**
** Copyright (c) 2005-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ----------------------------Description------------------------------
** 001 GES 20050314 @20313 Created initial version that provides support for command line
** processing based on a root node list provided in an XML file or via
** command line arguments. This is an alternate, special-purpose
** driver for the pattern engine.
** 002 GES 20050616 @21334 Updates to use a modified pattern engine interface that fully
** enables using the same profile and pattern engine instance for
** multiple invocations of the walk (one for each root node).
** Previously, this was broken but now it works.
** 003 GES 20060130 @24137 Moved core logic into a callable method and added support for debug
** level. Simplified the command line.
** 004 GES 20090429 @42059 Match package and class name changes.
** 005 CA 20140313 Moved to a graph DB implementation. When call-sites are found, uses
** the external-program AST as source and the edge between the source
** AST and the target is annotated with call-site info.
** 006 CA 20140408 Fixed filename matching for source-code originating from a different
** OS than the one where conversion is ran.
** 007 CA 20140715 The schema files are loaded from the data/namespace folder, not from
** the project basepath.
** 008 GES 20170424 Changes for call graph analysis v3.
** 009 CA 20180413 Made the GraphML output optional (via the '-g' argument).
** CA 20180503 Fixed graph access for update mode.
** 010 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 011 TJD 20240221 Updated calls to Graph serialization into graphml
** 012 GBB 20240826 Moving FileSystemDaemon to osresource package.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.uast;
import java.io.*;
import java.util.*;
import java.util.logging.*;
import com.goldencode.ast.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.pattern.*;
import com.goldencode.p2j.util.osresource.*;
import org.apache.tinkerpop.gremlin.structure.*;
import org.apache.tinkerpop.gremlin.process.traversal.*;
/**
* Special-purpose driver for the pattern engine to create a new call graph for one or more root
* entry points. This provides a command line interface, see {@link #main} for the syntax.
*/
public class CallGraphGenerator
{
/** Logger. */
private static final ConversionStatus LOG = ConversionStatus.get(CallGraphGenerator.class);
/**
* Update the ambiguous external programs and the explicit list of external programs provided
* as argument, by re-checking if the ambiguous call-sites can be disambiguated from the hints.
*
* @param rootNodes
* List of Progress AST filenames to use as root nodes or
* <code>null</code> to use the projects defined root node list
* (see {@link RootNodeList}).
* @param debug
* Debug level.
*/
public static void updateGraphs(String[] rootNodes, int debug)
throws ConfigurationException,
AstException
{
CallGraphWorker.getGraph();
PatternEngine.setDebugLevel(debug);
Iterator<Vertex> walker = CallGraphWorker.newUpdateCallGraphIterator(rootNodes, false);
walkTargets(walker, "callgraph/generate_call_graph");
walker = CallGraphWorker.newUpdateCallGraphIterator(rootNodes, false);
walkTargets(walker, "callgraph/generate_call_graph_post");
}
/**
* Implements the core of the call graph generation logic, a call graph
* will be generated for each of the root nodes passed. For a root node
* named <code>filename.p.ast</code>, the resulting call graph will be
* named <code>filename.p.ast.graph</code>.
*
* @param rootNodes
* List of Progress AST filenames to use as root nodes or
* <code>null</code> to use the projects defined root node list
* (see {@link RootNodeList}).
* @param debug
* Debug level.
*/
public static void generateGraphs(String[] rootNodes, int debug)
throws ConfigurationException,
AstException,
IOException
{
String folder = Configuration.getParameter("callgraph-db-folder",
CallGraphHelper.DEFAULT_CALLGRAPH_DB_FOLDER);
File f = new File(folder);
if (f.exists())
{
FileSystemDaemon.delete(f, true);
}
CallGraphWorker.getGraph();
PatternEngine.setDebugLevel(debug);
boolean projectRelative = false;
if (rootNodes == null)
{
// read root node list from XML
RootNodeList rnl = new RootNodeList();
rootNodes = rnl.getList();
projectRelative = true;
}
String basePath = Configuration.getParameter("basepath", ".");
PatternEngine pe = new PatternEngine(basePath, "*.ast");
// Initialize the graph by loading the entire code-set. For each added external program,
// create its include graph, too. This is temporary until TRPL is changed so that is backed
// by a graph DB: at that point, this step will no longer be necessary.
System.out.println("Started loading the code-set.");
pe.run("callgraph/load_code_set");
pe.run("callgraph/load_code_set_post");
System.out.println("Done loading the code-set.");
// mark all the programs referenced as triggers by the schema; this is similar to how
// RUN filename. is disambiguated
pe = new PatternEngine("data/namespace", "*.schema");
System.out.println("Started loading the schema-triggers.");
pe.run("callgraph/load_schema_triggers");
System.out.println("Done loading the schema-triggers.");
Iterator<Vertex> walker = CallGraphWorker.newCallGraphIterator(rootNodes, projectRelative);
walkTargets(walker, "callgraph/generate_call_graph");
walker = CallGraphWorker.newCallGraphIterator(rootNodes, projectRelative);
walkTargets(walker, "callgraph/generate_call_graph_post");
}
/**
* Walk the targets specified by the given iterator, in batches, until the iterator no longer
* provides any new targets.
*
* @param walker
* The target provider.
* @param profile
* The profile to be applied against the targets.
*/
private static void walkTargets(Iterator<Vertex> walker, String profile)
throws AstException,
ConfigurationException
{
PatternEngine pe = null;
while (walker.hasNext())
{
pe = new PatternEngine();
while (walker.hasNext())
{
Vertex what = walker.next();
// this is restricted only to external programs for now, so is OK to search for a
// filename
String name = what.<String>property("os-filename").value();
// gather all the AST files, to apply the rules in batches.
pe.addTarget(name + ".ast");
// TODO: when we an internal entry is targeted, we will need to walk the AST tree
// rooted at the internal entry, not the entire external program. for this case, it
// is needed to 1. load the external AST into memory 2. position on the AST targeted
// by the specified ID.
}
pe.run(profile);
}
}
/**
* Emit a syntax statement and optional error message to
* <code>stderr</code>, then exit the process with the specified return
* code. This method is called from {@link #main} when the user passes
* command line parameters which are not viable.
*
* @param msg
* Optional error message; may be <code>null</code>.
* @param rc
* Process return code used with <code>System.exit()</code>.
*/
private static void syntax(String msg, int rc)
{
if (msg != null)
{
LOG.log(Level.INFO, msg);
}
String[] staticText =
{
"Syntax:",
"",
" java CallGraphGenerator [-g] [-Dn] [-u] [root_node_list]",
"",
"Where",
"",
" options:",
" -g persist the graph in GraphML format, in file callgraph.graphml",
" -Dn set the debug level to 'n' (0 = none, 1 = status,",
" 2 = debug, 3 = trace)",
" -u sets the callgraph processing in update mode, where " +
" only the ambiguous nodes and their newly-resolved " +
" targets are processed. The targets specified in the " +
" root_nodes_list are ensured to be updated and must " +
" already exist in the callgraph.",
"",
" root_node_list = list of absolute and/or relative Progress",
" AST file names to use as root nodes (e.g.",
" filename.p.ast). These will be marked as" +
" entry-points only if not in update mode.",
"",
"By default, the debug level is set to status (1) and the",
"project's defined root node list is used."
};
for (int i = 0; i < staticText.length; i++)
{
LOG.log(Level.INFO, staticText[i]);
}
System.exit(rc);
}
/**
* Provides a command line interface for an end user to drive the pattern
* engine for the purpose of building one or more call graphs.
* <p>
* Syntax:
* <pre>
* java CallGraphGenerator [-Dn] [-u] [root_nodes...]
* </pre>
* Where:
* <ul>
* <li> -g saves the graph in GraphML format</li>
* <li> -Dn sets the pattern engine debug level where <code>n</code> is
* the numeric setting of the level (default is 1):
* <ul>
* <li> none (0)
* <li> status (1)
* <li> debug (2)
* <li> trace (3)
* </ul>
* <li>-u sets the callgraph processing in update mode, where only the ambiguous nodes and
* their newly-resolved targets are processed. The targets specified in the root_nodes
* list are ensured to be updated and must already exist in the callgraph.
*
* <li> [root_nodes...] is one or more root node filenames to process
* (which must be valid relative or absolute names based on the
* current directory)
* </ul>
* <p>
* Debug level is set to status (1) by default.
* <p>
* If no root nodes are listed on the command line, then the project's
* root node list will be used ({@link RootNodeList}).
*
* @param args
* List of command line arguments.
*/
public static void main(String[] args)
{
String[] list = null;
int debug = PatternEngine.MSG_STATUS;
int idx = 0;
boolean update = false;
// argument processing (all args are optional)
if (args.length > 0)
{
if (args[idx].equalsIgnoreCase("-g"))
{
CallGraphWorker.getGraph().traversal().io("callgraph.graphml").with(IO.writer, IO.graphml).write().iterate();
return;
}
// display help
if (args[idx].equals("?") || args[idx].equals("-?"))
{
// this doesn't return
syntax(null, 0);
}
// check for debug option
if (args[idx].toLowerCase().startsWith("-d"))
{
char level = args[idx].charAt(2);
if (Character.isDigit(level))
{
debug = Character.digit(level, 10);
if (debug < PatternEngine.MSG_NONE ||
debug > PatternEngine.MSG_TRACE)
{
syntax("Invalid debug level.", -1);
}
}
else
{
syntax("Debug level must be a numeric digit from " +
PatternEngine.MSG_NONE + " - " +
PatternEngine.MSG_TRACE + ".",
-2);
}
idx++;
}
// the idx value may have increased above
if (args.length > idx && args[idx].toLowerCase().equals("-u"))
{
update = true;
idx++;
}
// process an explicit list of root nodes
if (args.length > idx)
{
List<String> strings = new ArrayList<>();
// any other args are filenames
for (int i = idx; i < args.length; i++)
{
strings.add(args[i]);
}
list = strings.toArray(new String[0]);
}
}
try
{
if (update)
{
updateGraphs(list, debug);
}
else
{
generateGraphs(list, debug);
}
}
catch (Exception excpt)
{
LOG.log(Level.SEVERE, "", excpt);
}
finally
{
CallGraphHelper.shutdownGraph();
}
}
}