ConversionData.java
/*
** Module : ConversionData.java
** Abstract : Main access to incremental conversion db-backed data.
**
** Copyright (c) 2020-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------Description----------------------------------
** 001 CA 20200412 Created initial version, for incremental conversion support.
** 002 CA 20200416 Reduce the memory footprint for incremental conversion.
** CA 20200508 OO class references are qualified only if 4GL qualifies them.
** CA 20200525 If an include file is missing, then reconvert the program referencing it.
** CA 20210423 'tmpTabNames' has as value an AST which may be from a different file. For incremental
** conversion, if we include a file which affects values in this map, then all affected
** keys must be considered as dependencies and their files reconverted, too.
** CA 20211012 Fixed generated WSDL and reworked SOAP support to rely on namespace, when resolving an
** operation.
** GES 20220502 Match API method name change.
** RAA 20230109 Changed inline statement(s) to prepared statement(s).
** 003 CA 20250515 Added 'connect()' to some methods where it was missing.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.convert.db;
import java.io.*;
import java.security.*;
import java.sql.*;
import java.util.*;
import java.util.function.*;
import com.goldencode.ast.*;
import com.goldencode.p2j.preproc.*;
import com.goldencode.p2j.uast.*;
/**
* Provides APIs to create 'any-purpose' db-backed collections and explicitly defined tables.
* <p>
* Currently, this supports creating {@link H2Map maps}, {@link H2Set sets}, and also maps which
* have as values another {@link H2MapToMap map} or a {@link H2MapToSet set}.
* <p>
* The associated tables for the db-backed collections are created on first access, and their
* name follow this rules:
* <ul>
* <li>main collection table will have the collection's name</li>
* <li>work table for the collection will have the <code>__work</code> suffix</li>
* <li>child main tables will have the <code>__map</code> or <code>__set</code> suffix</li>
* <li>work table for the child main table will have the <code>_work</code> suffix</li>
* </ul>
* <p>
* Changes to these collections will be logged in the work tables, and if the collection already
* exists in the database, will be restored from the work table. This allows to 'restore history'
* during incremental conversion, by clearing from the work table all changes belonging to the
* files currently being converted.
*/
public class ConversionData
{
/** The helper for SQL access to the database. */
private static final DBHelper helper = new DBHelper();
/**
* Clean all records about this file, from the incremental conversion database.
* <p>
* This will clear all work tables which have records for this file AST ID, and all
* records from the <code>CLASS_DEFINITIONS</code> table.
*
* @param file
* The file to clear.
*
* @return The set of dependency files.
*/
public static Set<String> clean(String file)
{
// go through tables and remove records for this specific file
String[] exts = { "", ".ast", ".p2o", ".schema", ".dict" };
Long[] fileIds = new Long[exts.length];
AstManager mgr = AstManager.get();
for (int i = 0; i < exts.length; i++)
{
long astId = mgr.getTreeId(file + exts[i]);
if (astId != AstManager.INVALID_ID)
{
fileIds[i] = astId;
}
}
// if there are other records which are dependent on an AST in this file, then the
// programs for those records need to be re-converted, too. this must be done in a recursive way,
// so the entire dependency graph is re-converted
// only 'tmpTabNodes' has AST values, so only this table will be checked for now
String[] astValueTables = { "tmpTabNodes" };
// get all records for this table which should be deleted, and
Set<Long> dependencies = new HashSet<>();
Arrays.asList(fileIds).stream().filter((l) -> l != null).forEach((l) -> dependencies.add(l));
for (String table : astValueTables)
{
// check if the table exists
boolean[] found = { false };
Object[] args = new Object[] {table.toUpperCase() + "__WORK"};
helper.executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES " +
"WHERE TABLE_SCHEMA = SCHEMA() AND TABLE_NAME LIKE ?", (rs) ->
{
found[0] = true;
}, args);
if (!found[0])
{
continue;
}
String sql = "select fileid, astid, pos, key, value from " + table + "__WORK";
Set<Long> remaining = new HashSet<>();
do
{
remaining.clear();
helper.executeQuery(sql, (rs) ->
{
try
{
AstExternalizble o = (AstExternalizble) rs.getObject("value");
long astId = mgr.getTreeId(o.getAstId());
if (dependencies.contains(astId))
{
remaining.add(rs.getLong("fileid"));
}
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
});
}
while (dependencies.addAll(remaining));
}
Consumer<ResultSet> c = (rs) ->
{
try
{
String workTableName = rs.getString(1);
String deleteSql = "delete from " + workTableName + " where fileid = ?";
for (Long fileId : fileIds)
{
if (fileId == null)
{
continue;
}
helper.executeUpdate(deleteSql, fileId);
}
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
};
helper.executeSQL("DELETE FROM CLASS_DEFINITIONS WHERE FILENAME = ?", file);
helper.executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES " +
"WHERE TABLE_SCHEMA = SCHEMA() AND TABLE_NAME LIKE '%__WORK'", c);
Set<String> depFiles = new HashSet<>();
dependencies.forEach((fileId) ->
{
String name = mgr.getTreeName(fileId);
for (String ext : exts)
{
if (!ext.isEmpty() && name.endsWith(ext))
{
name = name.substring(0, name.length() - ext.length());
break;
}
}
depFiles.add(name);
});
return depFiles;
}
/**
* Clean all tables from the database.
*/
public static void cleanAll()
{
connect();
// clean the entire database
helper.executeSQL("DROP ALL OBJECTS");
disconnect();
}
/**
* Disconnect the database.
* <p>
* This will persist all db-backed collection.
*/
public static void disconnect()
{
if (!connected())
{
return;
}
helper.disconnect();
}
/**
* Connect to the database.
* <p>
* This will create the <code>class_definitions</code> and <code>file_signature</code> tables,
* if they do not already exist.
*/
public static void connect()
{
if (connected())
{
return;
}
helper.connect();
// create 'static' tables
// filename fileid qname pkgname classname isbuiltin isdotnet isjava
helper.executeSQL("CREATE TABLE IF NOT EXISTS class_definitions (" +
" filename VARCHAR PRIMARY KEY," + // TODO: case sensitivity?
" qname VARCHAR," +
" jpkgname VARCHAR," +
" jclassname VARCHAR," +
" isbuiltin BOOLEAN," +
" isdotnet BOOLEAN," +
" isjava BOOLEAN);");
helper.executeSQL("CREATE UNIQUE INDEX IF NOT EXISTS idx_class_definitions_qname ON class_definitions (qname);");
helper.executeSQL("CREATE UNIQUE INDEX IF NOT EXISTS idx_class_definitions_filename ON class_definitions (filename);");
// create the program and include files MD5 table and 'with_services' flag.
helper.executeSQL("CREATE TABLE IF NOT EXISTS file_signature (" +
" filename VARCHAR PRIMARY KEY," +
" with_services BOOLEAN," +
" md5 VARCHAR);");
}
/**
* Check if the database is connected.
*
* @return See above.
*/
public static boolean connected()
{
return helper.connected();
}
/**
* Get the temp-idx value for the specified AST, in the <code>ooVarTempIdx__work</code> table.
*
* @param astId
* The temp index value.
*
* @return The found temp idx, or -1.
*/
public static int getTempIdx(long astId)
{
connect();
String table = "ooVarTempIdx__work";
String sql = "select key from " + table + " where fileId = ? and astId = ?";
long fileId = AstManager.get().getTreeId(astId);
List<Object[]> res = helper.executeQuery(sql, fileId, astId);
if (res.isEmpty())
{
return -1; //throw new RuntimeException("Can't resolve the tempidx for " + astId + "!");
}
String val = (String) res.get(0)[0];
int tempIdx = Integer.parseInt(val.substring(val.lastIndexOf('-') + 1));
return tempIdx;
}
/**
* Compute the MD5 hash for the specified file.
*
* @param file
* The file name.
*
* @return The MD5 hash.
*/
public static String computeFileHash(String file)
{
try
{
InputStream fis = new FileInputStream(file);
byte[] buffer = new byte[1024];
MessageDigest complete = MessageDigest.getInstance("MD5");
int numRead;
do
{
numRead = fis.read(buffer);
if (numRead > 0)
{
complete.update(buffer, 0, numRead);
}
}
while (numRead != -1);
fis.close();
byte[] checksum = complete.digest();
String result = "";
for (int i = 0; i < checksum.length; i++)
{
result += Integer.toString((checksum[i] & 0xff) + 0x100, 16).substring(1);
}
return result;
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
/**
* Compute and save the MD5 hash for this file.
*
* @param file
* The file name.
*/
public static void saveFileHash(String file)
{
connect();
String insertOrUpdate = "MERGE INTO file_signature KEY (filename) VALUES (?, ?, ?)";
String hash = computeFileHash(file);
helper.executeSQL(insertOrUpdate, file, false, hash);
// check the hint files
try
{
PreprocessorHints hints = new PreprocessorHints(file);
Set<String> includes = hints.getUniqueIncludes();
for (String finclude : includes)
{
hash = computeFileHash(finclude);
helper.executeSQL(insertOrUpdate, finclude, false, hash);
}
}
catch (PreprocessorException e)
{
throw new RuntimeException(e);
}
}
/**
* Check if this file must be converted.
* <p>
* A file must be converted if the file is new, or itself or any of its include files
* (from reading the associated <code>.pphints</code> has changed.
*
* @param file
* The file name.
*
* @return <code>true</code> if this file has changed and must be converted.
*/
public static boolean mustConvertFile(String file)
{
File fast = new File(file + ".ast");
if (!fast.exists())
{
// new file
return true;
}
Function<String, String> fdbhash = (fname) ->
{
// check the file's MD5
List<Object[]> res = helper.executeQuery("select md5 from file_signature where filename = ?", fname);
if (res.isEmpty())
{
return null;
}
return (String) res.get(0)[0];
};
String currentHash = fdbhash.apply(file);
String hash = computeFileHash(file);
if (!hash.equals(currentHash))
{
return true;
}
// check the hint files
try
{
PreprocessorHints hints = new PreprocessorHints(file);
Set<String> includes = hints.getUniqueIncludes();
for (String finclude : includes)
{
if (!new File(finclude).exists())
{
// the include file no longer exists...
return true;
}
currentHash = fdbhash.apply(finclude);
hash = computeFileHash(finclude);
if (!hash.equals(currentHash))
{
return true;
}
}
}
catch (PreprocessorException e)
{
throw new RuntimeException(e);
}
return false;
}
/**
* Get the next value for the specified sequence.
*
* @param name
* The sequence name.
*
* @return The next sequence value.
*/
public static int nextSequenceValue(String name)
{
connect();
String seqName = name + "__seq";
helper.executeSQL("CREATE SEQUENCE IF NOT EXISTS " + seqName);
return helper.executeCount("select next value for " + seqName);
}
/**
* Mark this file 'with_services'.
*
* @param file
* The file name.
*/
public static void withServicesFile(String file)
{
// mark this file 'with services'
connect();
String update = "UPDATE file_signature SET with_services = true WHERE filename = ?";
helper.executeSQL(update, file);
}
/**
* Get all program files which are marked 'with_services'.
*
* @return See above.
*/
public static List<String> listWithServicesFiles()
{
connect();
List<String> files = new ArrayList<>();
String sql = "SELECT filename FROM file_signature WHERE with_services";
helper.executeQuery(sql, (rs) -> {
try
{
files.add(rs.getString(1));
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
});
return files;
}
/**
* Get the simple java class names for all classes in this package (builtin or converted code).
*
* @param pack
* The package name.
*
* @return The set of simple java class names.
*/
public static Set<String> getClassesInPackage(String pack)
{
connect();
Set<String> res = new HashSet<>();
String select = "select jclassname from class_definitions where jpkgname = ?";
Consumer<ResultSet> c = (rs) ->
{
try
{
res.add(rs.getString(1));
}
catch (SQLException e)
{
throw new RuntimeException(e);
}
};
helper.executeQuery(select, c, pack);
return res;
}
/**
* Update the <code>class_definitions</code> table with the converted Java class simple name
* and package.
*
* @param filename
* The converted class file name.
* @param jpkgname
* The converted Java package name.
* @param jclassname
* The converted simple Java class name.
*/
public static void setConvertedClassName(String filename, String jpkgname, String jclassname)
{
connect();
String update = "update class_definitions set jpkgname = ?, jclassname = ? where filename = ?";
helper.executeUpdate(update, jpkgname, jclassname, filename);
}
/**
* List all legacy builtin classes which are registered in the database.
*
* @return A list of qualified legacy class names.
*/
public static List<String> listLegacyClasses()
{
connect();
String select = "select qname from class_definitions where isbuiltin order by qname";
List<Object[]> qres = helper.executeQuery(select);
List<String> res = new ArrayList<>();
qres.forEach(q -> res.add((String) q[0]));
return res;
}
/**
* Find the associated qualified class name for this file, defining a legacy class.
*
* @param filename
* The file name.
*
* @return The qualified class name, or <code>null<code> if it does not exist.
*/
public static String findClassQName(String filename)
{
connect();
String select = "select qname from class_definitions where filename = ?";
List<Object[]> res = helper.executeQuery(select, filename);
if (res.isEmpty())
{
return null;
}
return (String) res.get(0)[0];
}
/**
* Find the associated file name for the specified qualified class name of a legacy class.
*
* @param qname
* The qualified class name.
*
* @return The file name, or <code>null<code> if it does not exist.
*/
public static String findClassFilename(String qname)
{
connect();
String select = "select filename from class_definitions where qname = ?";
List<Object[]> res = helper.executeQuery(select, qname.toLowerCase());
if (res.isEmpty())
{
return null;
}
return (String) res.get(0)[0];
}
/**
* Get the converted Java package and simple class name, for this file defining a legacy class.
*
* @param filename
* The file name.
*
* @return The converted Java package and simple class name, or <code>null</code> if it does
* not exist.
*/
public static Object[] getConvertedClassName(String filename)
{
connect();
String select = "select jpkgname, jclassname " +
"from class_definitions where filename = ?";
List<Object[]> res = helper.executeQuery(select, filename);
if (res.isEmpty())
{
return null;
}
return res.get(0);
}
/**
* Get the converted simple class name for the specified fully-qualified 4GL class name.
*
* @param qname
* The qualified class name.
*
* @return The simple Java class name or <code>null</code> if it was not found.
*/
public static String getConvertedSimpleClassName(String qname)
{
if (qname == null)
{
return null;
}
connect();
qname = qname.toLowerCase();
String select = "select jclassname from class_definitions where qname = ?";
List<Object[]> res = helper.executeQuery(select, qname);
if (res.isEmpty())
{
return null;
}
return (String) res.get(0)[0];
}
/**
* Save the specified class definition details into the <code>class_definitions</code> table.
*
* @param clsDef
* The legacy class details.
*/
public static void saveClass(ClassDefinition clsDef)
{
connect();
String filename = clsDef.getFilename();
String qname = clsDef.getName();
String jpkgname = clsDef.getJavaPackage();
String jclassname = clsDef.getSimpleJavaName();
String insertSql = "MERGE INTO class_definitions KEY (filename) values (?, ?, ?, ?, ?, ?, ?)";
helper.executeSQL(insertSql,
filename,
qname.toLowerCase(),
jpkgname,
jclassname,
clsDef.isBuiltIn(),
clsDef.isDotNet(),
clsDef.isJava());
}
/**
* Create a new db-backed set with the specified key class and name.
*
* @param keyClass
* The key class.
* @param name
* The collection name.
*
* @return The associated {@link H2Set}.
*/
public static Set<?> createSet(Class<?> keyClass, String name)
{
connect();
H2Set set = (H2Set) helper.getCollection(name);
if (set == null)
{
set = new H2Set(keyClass, name, helper);
helper.register(name, set);
}
return set;
}
/**
* Create a new db-backed map, which has as values another map.
*
* @param keyClass
* The key class.
* @param key2Class
* The child's map key class.
* @param valueClass
* The child's map value class.
* @param name
* The collection name.
*
* @return The associated {@link H2MapToMap}.
*/
public static Map<?, ?> createMapToMap(Class<?> keyClass,
Class<?> key2Class,
Class<?> valueClass,
String name)
{
connect();
H2MapToMap map = (H2MapToMap) helper.getCollection(name);
if (map == null)
{
map = new H2MapToMap<>(keyClass, key2Class, valueClass, name, helper);
helper.register(name, map);
}
return map;
}
/**
* Create a new db-backed map, which has as values another set.
*
* @param keyClass
* The key class.
* @param valueClass
* The child's set key class.
* @param name
* The collection name.
*
* @return The associated {@link H2MapToSet}.
*/
public static Map<?, ?> createMapToSet(Class<?> keyClass, Class<?> valueClass, String name)
{
connect();
H2MapToSet map = (H2MapToSet) helper.getCollection(name);
if (map == null)
{
map = new H2MapToSet<>(keyClass, valueClass, name, helper);
helper.register(name, map);
}
return map;
}
/**
* Create a new db-backed map, with the specified key and value class.
*
* @param keyClass
* The key class.
* @param valueClass
* The child's map value class.
* @param name
* The collection name.
*
* @return The associated {@link H2Map}.
*/
public static Map<?, ?> createMap(Class<?> keyClass, Class<?> valueClass, String name)
{
connect();
H2Map map = (H2Map) helper.getCollection(name);
if (map == null)
{
map = new H2Map(keyClass, valueClass, name, helper);
helper.register(name, map);
}
return map;
}
/**
* Compact the elements of this collection, to reduce the memory footprint.
*
* @param name
* The collection name.
* @param file
* The AST file which has finished processing.
*/
public static void compact(String name, String file)
{
StoredCollection c = helper.getCollection(name);
if (c != null)
{
AstManager mgr = AstManager.get();
long fileId = mgr.getTreeId(file);
((StoredCollection) c).compact(fileId);
}
}
/**
* Check if we are currently persisting the db-backed collections.
*
* @return See above.
*/
static boolean inPersist()
{
return helper.inPersist();
}
}