DynamicValidationHelper.java
/*
** Module : DynamicValidationHelper.java
** Abstract : Helper class for dynamic validation expressions.
**
** Copyright (c) 2017-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
** 001 SVL 20171024 Created initial version.
** 002 ECF 20190827 Fixed SchemaDictionary initialization in the event "default-databases" is configured.
** 003 CA 20200930 Use a SymbolResolver exemplar to create the instance used by runtime conversion.
** OM 20210309 Do not use DmoMeta as key in TableMapper because temp-tables share the same DmoMeta.
** ECF 20210914 Fixed schema dictionary temp-table scope caching.
** CA 20220918 Replaced LFUAgingCache with LRUCache.
** SVL 20230113 Improved performance by replacing some "for-each" loops with indexed "for" loops.
** SVL 20230113 Fixed compilation error.
** HC 20230116 Replaced some handle usages with the actual wrapped resources for
** performance.
** CA 20230116 Avoid using handle.unwrap, handle.getReference or other BDT usage from within FWD runtime.
** 004 OM 20230215 Handled collision of buffers with same dynamic table name.
** 005 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 006 DDF 20240216 cache can only be created through CacheManager now and has been made non-final.
** 007 OM 20240318 API changes in DynamicConversionHelper.
** 008 AL2 20240530 Prepare the dictFile as well to be able to work with SchemaWorker in run-time conversion.
** 009 AL2 20240613 Cleared dictFile once the dynamic validation is done.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.persist;
import antlr.*;
import com.goldencode.ast.*;
import com.goldencode.cache.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.schema.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.uast.*;
import com.goldencode.p2j.ui.*;
import com.goldencode.p2j.util.ErrorManager;
import com.goldencode.p2j.util.logging.*;
import java.io.*;
import java.util.*;
import java.util.concurrent.atomic.*;
import java.util.logging.*;
/**
* Helper class for dynamic validation expressions conversion.
*/
public class DynamicValidationHelper
extends DynamicConversionHelper
{
/** Counter for converted validation expressions. */
private static final AtomicLong valexpCounter = new AtomicLong(1);
/** Name of the variable which replaces the validated field reference in the validation expression. */
public static final String VAL_EXPR_VAR_NAME = "valExprDynParam";
/** The name of the generated java class. */
private static final String DYN_GEN_VAL_EXPR_CLASS = "DynGenValExpr";
/** Logger. */
private static final CentralLogger LOG = CentralLogger.get(DynamicValidationHelper.class.getName());
/** Context-local data. */
private static final ContextLocal<WorkArea> local = new ContextLocal<WorkArea>()
{
protected WorkArea initialValue()
{
return new WorkArea();
}
};
/**
* Cache for JASTs generated from validation expressions. Cache management uses an aging and
* usage algorithm to chose entries to be dropped when the maximum capacity is reached.
*/
private static ExpiryCache<ValExprCacheKey, JavaAst> cache;
/**
* Initializes the cache using CacheManager. The default value of the cache
* is used when there is no size available from the configuration.
*/
public static void initializeCache()
{
cache = CacheManager.createLRUCache(DynamicValidationHelper.class, 65536);
cache.addCacheExpiryListener(event ->
{
// debug only: to see when the cache is full
if (LOG.isLoggable(Level.INFO))
{
LOG.log(Level.INFO, "DynamicValidationHelper cache is full. Dumped " +
event.getExpiredEntries().size() + " old entries.");
}
});
}
/**
* Worker for parsing a given validation expression string and returning a
* {@link JastValidationExpr} instance.
*
* @param query
* Query which iterates the records in the validated buffer.
* @param validatedBuffer
* Validated buffer.
* @param validatedProperty
* Property name of the validated field.
* @param validateExpression
* Validation expression.
*
* @return {@link JastValidationExpr} instance representing the given validation expression.
*/
public static JastValidationExpr parseValidationExpression(QueryWrapper query,
RecordBuffer validatedBuffer,
String validatedProperty,
String validateExpression)
{
ValExprCacheKey cacheKey = new ValExprCacheKey(validateExpression, validatedBuffer, validatedProperty);
int numBuffers = query.numBuffers().intValue();
ArrayList<Buffer> buffers = new ArrayList<>(numBuffers);
for (int i = 1; i <= numBuffers; i++)
{
buffers.add(query.getBufferByIndex(i));
}
JavaAst javaAst;
synchronized (cache)
{
// check the cache for a previously constructed JAST
javaAst = cache.get(cacheKey);
}
if (javaAst == null)
{
long t0 = System.nanoTime(); // start the timer
WorkArea wa = locate();
long valExprId = valexpCounter.getAndIncrement();
AstManager astManager = AstManager.get();
// set the info about what we are currently parsing
wa.valExpr = validateExpression;
wa.buffers = buffers;
wa.dynExprFile = "." + File.separator + "in-mem-valexp" + valExprId + ".p";
wa.jastFile = wa.dynExprFile + ".jast";
wa.astFile = wa.dynExprFile + ".ast";
wa.p2oFile = wa.dynExprFile + ".p2o";
wa.dictFile = wa.dynExprFile + ".dict";
try
{
StringBuilder code = new StringBuilder();
SymbolResolver sym = SymbolResolver.newRuntimeInstance(); // build the symbol resolver
SchemaDictionary dict = RecordBuffer.getSchemaDictionary(buffers);
sym.setSchemaDictionary(dict); // set the schema dictionary used by this context
Map<Buffer, String> tempTableNames = new HashMap<>();
try
{
// load p2o and schema definitions for all temporary buffers
try
{
// load the p2o and schema data
prepareDynamicTables(dict, code, wa.p2oFile, wa.dictFile, wa.buffers, tempTableNames);
}
catch (PersistenceException exc)
{
displayConversionError(exc, "prepare the temp-table buffers for", code.toString());
return null;
}
// add a default scope that represents the scope of the external procedure
sym.addSchemaScope(false);
// form converted code
code.append("return ");
code.append(validateExpression);
code.append(".");
boolean complete;
Exception err = null;
String codeString = code.toString();
StringReader in = new StringReader(codeString);
ProgressLexer lexer = new ProgressLexer(in, sym);
ProgressParser parser = new ProgressParser(lexer, sym);
try
{
// throw exception instead of print event to console
parser.setConsumeError(false);
complete = parser.external_proc();
}
catch (RecognitionException | TokenStreamException exc)
{
err = exc;
complete = false;
}
catch (RuntimeException exc2)
{
// if the RecognitionException was re-thrown wrapped as RuntimeException
// from ProgressParser.reportError() we need to get the original back
if (exc2.getCause() instanceof RecognitionException)
{
err = (RecognitionException) exc2.getCause();
complete = false;
}
else
{
// rethrow the exception of unknown cause
throw exc2;
}
}
if (!complete)
{
displayConversionError(err, "parse", codeString);
return null;
}
try
{
ProgressAst pAst = (ProgressAst) parser.getAST();
pAst.brainwash(wa.dynExprFile, true);
// prepare the initial AST
prepareTree(pAst, wa.buffers, wa.dynExprFile, DYN_GEN_VAL_EXPR_CLASS, tempTableNames);
TableMapper.LegacyFieldInfo lfi = TableMapper.getLegacyFieldInfo(validatedBuffer,
validatedProperty);
String validatedField = lfi.getLegacyName();
// get the name of the validate buffer
StringBuilder sb = new StringBuilder();
if (!validatedBuffer.isTemporary())
{
sb.append(validatedBuffer.getDatabase().getName()).append(".");
}
sb.append(validatedBuffer.getLegacyName());
String validateBufferName = sb.toString().toLowerCase();
replaceParams(pAst, validateBufferName, validatedField);
// run conversion
ConversionPool.runTask(ConversionProfile.ANNOTATIONS, pAst);
pAst = (ProgressAst) astManager.loadTree(wa.astFile);
ConversionPool.runTask(ConversionProfile.BASE_STRUCTURE, pAst);
pAst = (ProgressAst) astManager.loadTree(wa.astFile);
ConversionPool.runTask(ConversionProfile.CORE_CONVERSION, pAst);
JavaAst jCode = (JavaAst) astManager.loadTree(wa.jastFile);
// DBG: String res1 = jCode.dumpTree(true));
// DBG: String res2 = ConversionPool.runTask(
// ConversionProfile.BREW, jCode).getStoredObject(DYN_GEN_VAL_EXPR_CLASS);
// TODO using ConversionProfile.POSTPROCESS_OPEN_QUERY may be an overkill,
// we can leave only the part necessary for dynamic validation expressions
ConversionPool.runTask(ConversionProfile.POSTPROCESS_OPEN_QUERY, jCode);
jCode = (JavaAst) astManager.loadTree(wa.jastFile);
ConversionPool.runTask(ConversionProfile.POSTPROCESS_VALIDATION_EXP, jCode);
javaAst = (JavaAst) astManager.loadTree(wa.jastFile);
// DBG: String res3 = ConversionPool.runTask(
// ConversionProfile.BREW, jCode).getStoredObject(DYN_GEN_VAL_EXPR_CLASS);
if (javaAst == null)
{
return null; // conversion failed? this is strange
}
// intern the AST strings for long term storage in the cache
javaAst.intern();
addToCache(cacheKey, javaAst);
}
catch (Exception exc)
{
if (exc.getCause() instanceof CompileException)
{
CompileException cex = (CompileException) exc.getCause();
ErrorManager.recordOrShowError(
cex.getNumber(), cex.getMessage(), true, cex.isPrefix());
}
else
{
displayConversionError(exc, "convert", codeString);
}
return null;
}
}
finally
{
// clean up the schema
sym.deleteSchemaScope(false);
// clean up the progress, java, and P2O ASTs, they are no longer needed
astManager.removeTree(wa.dynExprFile);
astManager.removeTree(wa.astFile);
astManager.removeTree(wa.jastFile);
astManager.removeTree(wa.p2oFile);
astManager.removeTree(wa.dictFile);
}
}
finally
{
// reset the WorkArea temporary data
wa.valExpr = null;
wa.buffers = null;
wa.dynExprFile = null;
wa.jastFile = null;
wa.astFile = null;
wa.p2oFile = null;
wa.dictFile = null;
if (LOG.isLoggable(Level.FINE))
{
long dt = (System.nanoTime() - t0) / 1000;
LOG.log(Level.FINE, "Dynamic-Expression-Timer = " + dt / 1000.0 + " ms");
}
}
}
else
{
if (LOG.isLoggable(Level.FINE))
{
LOG.log(Level.FINE, "Found in cache " + cacheKey + "\n" + javaAst.dumpTree());
}
}
return new JastValidationExpr(javaAst, buffers);
}
/**
* Replace field references in the Progress code AST:
* <ul>
* <li>references to the validated field are replaced with a variable which is used as a
* parameter when executing validation expression;</li>
* <li>reference to buffer's fields other than the currently validated field are replaced with
* unknown values, warning is shown;</li>
* <li>reference to fields of other buffers raise error.</li>
* </ul>
*
* @param progressAst
* Progress code AST to be modified.
* @param validatedBufferName
* Fully qualified legacy name of the validated buffer.
* @param validatedField
* Legacy name of the validated field.
*/
private static void replaceParams(ProgressAst progressAst,
String validatedBufferName,
String validatedField)
{
String validatedSchemaName = validatedBufferName + "." + validatedField.toLowerCase();
ArrayList<ProgressAst> unknownNodes = null;
List<ProgressAst> varNodes = new ArrayList<>();
Iterator<Aast> iter = progressAst.iterator();
while (iter.hasNext())
{
ProgressAst next = (ProgressAst) iter.next();
int type = next.getType();
if (type > ProgressParserTokenTypes.BEGIN_FIELDTYPES &&
type < ProgressParserTokenTypes.END_FIELDTYPES)
{
String bufname = (String) next.getAnnotation("bufname");
String schemaname = (String) next.getAnnotation("schemaname");
if (!bufname.equals(validatedBufferName))
{
// wrapped with RuntimeException in order to be handled with other compile
// exceptions in parseValidationExpression
throw new RuntimeException(new CompileException(-1,
"Progress cannot reference fields other than fields of the validated buffer " +
"in a dynamic validation statement. Cannot reference field " + schemaname +
". Validation statement: " + locate().valExpr,
next));
}
else if (!schemaname.equals(validatedSchemaName))
{
ErrorManager.recordOrShowWarning(-1,
"WARNING: in a dynamic validation statement values of the validated buffer's " +
"fields other than the currently validated field are UNKNOWN. Replacing " +
"reference to " + schemaname + " field with the unknown literal. Validation " +
"statement: " + locate().valExpr,
false, false, false, false);
if (unknownNodes == null)
{
unknownNodes = new ArrayList<>();
}
unknownNodes.add(next);
}
else
{
varNodes.add(next);
}
}
}
if (unknownNodes != null)
{
for (int i = 0; i < unknownNodes.size(); i++)
{
ProgressAst node = unknownNodes.get(i);
ProgressAst unknown = new ProgressAst();
unknown.setType(ProgressParserTokenTypes.UNKNOWN_VAL);
unknown.setText("?");
node.getParent().graftAt(unknown, node.getIndexPos());
node.remove();
}
}
if (varNodes.size() > 0)
{
injectVariable(varNodes.toArray(new ProgressAst[varNodes.size()]), VAL_EXPR_VAR_NAME, progressAst);
}
else
{
ErrorManager.recordOrShowWarning(-1,
"WARNING: validation statement '" + locate().valExpr + "' for " + validatedField +
" doesn't contain the field itself",
false, false, false, false);
}
}
/**
* Add generated JAST to the cache.
*
* @param cacheKey
* Cache key.
* @param javaAst
* Generate JAST.
*/
private static void addToCache(ValExprCacheKey cacheKey, JavaAst javaAst)
{
synchronized (DynamicValidationHelper.cache)
{
JavaAst racer = DynamicValidationHelper.cache.get(cacheKey);
if (racer == null)
{
// now we can store it for subsequent use
DynamicValidationHelper.cache.put(cacheKey, javaAst);
if (LOG.isLoggable(Level.FINE))
{
LOG.log(Level.FINE, "Added to cache as " + cacheKey + "\n" + javaAst.dumpTree());
}
}
else
{
if (LOG.isLoggable(Level.INFO))
{
LOG.log(Level.INFO,
"Query was processed in a parallel thread [" + cacheKey.validateExpression + "].");
}
}
}
}
/**
* Display conversion error.
*
* @param exc
* Exception caused by the conversion error.
* @param operation
* Name of the conversion stage at which the error occurred.
* @param progressCode
* Progress code being converted.
*/
private static void displayConversionError(Exception exc, String operation, String progressCode)
{
WorkArea wa = locate();
String msg = "Could not " + operation + " the '" + wa.valExpr + "' validation expression";
ErrorManager.recordOrShowError(-1, msg, true, false, false);
if (LOG.isLoggable(Level.SEVERE))
{
// print the important message on SEVERE level
LOG.log(Level.SEVERE,
"Failed to convert P4GL validation expression code: '" + progressCode + "'\n" +
msg);
if (LOG.isLoggable(Level.FINE))
{
// display the full stack on a lower log level
LOG.log(Level.FINE, "", exc);
}
}
}
/**
* Locate the context-local {@link WorkArea} instance.
*
* @return See above.
*/
private static WorkArea locate()
{
return local.get();
}
/**
* Container of context-local data.
*/
private static class WorkArea
{
/** The validation expression being converted. */
private String valExpr = null;
/** The buffers used by the validation expression. */
private ArrayList<Buffer> buffers = null;
/** The file name of the class containing the converted validation expression code. */
private String dynExprFile = null;
/** The name of the generated {@link JavaAst} tree. */
private String jastFile = null;
/** The name of the converted {@link ProgressAst} tree. */
private String astFile = null;
/**
* The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the temp-table
* objects for this validation expression.
*/
private String p2oFile = null;
/** The name of the {@link ProgressAst} tree with the temp-table objects for schema worker. */
private String dictFile = null;
}
/**
* Class that keys the JAST tree generated from the validation expression.
*/
private static class ValExprCacheKey
{
/** Validation expression. */
private final String validateExpression;
/** DMO class of the validated buffer. */
private final Class validatedBufferDMOClass;
/** Property name of the validate field. */
private final String validatedProperty;
/** Hash code. */
private final int hashCode;
/**
* Default constructor.
*
* @param validateExpression
* Validation expression.
* @param rbuf
* Validated record buffer.
* @param validatedProperty
* Property name of the validate field.
*/
ValExprCacheKey(String validateExpression, RecordBuffer rbuf, String validatedProperty)
{
this.validateExpression = validateExpression;
this.validatedBufferDMOClass = rbuf.getDMOImplementationClass();
this.validatedProperty = validatedProperty;
// pre-compute hash code
int result = validateExpression.hashCode();
result = 31 * result + validatedBufferDMOClass.hashCode();
hashCode = 31 * result + validatedProperty.hashCode();
}
/**
* Checks whether some other object is "equal to" this one.
*
* @param o
* Another object to compare to.
*
* @return {@code true} if this object is the same as the obj argument.
*/
@Override
public boolean equals(Object o)
{
if (this == o)
{
return true;
}
if (!(o instanceof ValExprCacheKey))
{
return false;
}
ValExprCacheKey that = (ValExprCacheKey) o;
return validateExpression.equals(that.validateExpression) &&
validatedBufferDMOClass.equals(that.validatedBufferDMOClass) &&
validatedProperty.equals(that.validatedProperty);
}
/**
* Get hash code for this cache key.
*
* @return hash code for this cache key.
*/
@Override
public int hashCode()
{
return hashCode;
}
/**
* Return a string representation of this object.
*
* @return string representation of this object.
*/
@Override
public String toString()
{
return "ValExprCacheKey{" +
"validateExpression=" + validateExpression +
", validatedBufferDMOClass=" + validatedBufferDMOClass +
", validatedProperty=" + validatedProperty +
"}";
}
}
}