DynamicQueryHelper.java
/*
** Module : DynamicQueryHelper.java
** Abstract : Helper class for dynamic queries.
**
** Copyright (c) 2013-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description----------------------------------------
** 001 CA 20130929 Created initial version.
** 002 SVL 20131115 Add "force_dmo_alias" and "is_dynamic_table" annotations for dynamic buffers.
** 003 CA 20131013 Added no-op deleted() method, required by the changes in Finalizable interface.
** 004 OM 20140122 Added support for parsing predicated and return of FindQuery -s needed by
** findFirst / findLast and findUnique methods of BufferImpl.
** 005 OM 20140128 Adding normalization when accessing TableMapper.getLegacyName().
** 006 SVL 20140320 loadPermanentSchemas loads schemas from resources. Removed unnecessary loading
** of registry.xml.
** 007 ECF 20140404 Reimplemented dynamic conversion to use ConversionPool.
** 008 OM 20140417 Added case-sensitive support.
** 009 ECF 20140425 Removed unused flag.
** 010 VMN 20140506 Added enhance schema name conversion support for hint "escape" attribute.
** 011 VMN 20140511 Fixed FIRST clause in preparePredicate.
** 012 SVL 20140601 Fixed emission of buffer name in H011.
** 013 VMN 20140708 Enhanced H002 fix: force_dmo_alias annotation is needed for both static and
** dynamic buffers, to ensure converted buffer names (DMO aliases) match the
** caller's DMO aliases.
** 014 ECF 20140918 Fixed memory leaks which were leaving ASTs behind in memory. Reduced footprint
** of P2OLookup instances by intern'ing string data.
** 015 VMN 20141121 Fixed dot and colon usages in predicate for QUERY-PREPARE() in order to
** handle 4GL behavior.
** 016 VMN 20141204 Fixed dot and colon usages when they are part of a string constant, embedded
** in the query predicate for QUERY-PREPARE().
** 017 ECF 20150105 Modified code to match SchemaDictionary.loadFromAst API signature change.
** 018 ECF 20150109 Normalize legacy field names when using them as hash map keys.
** 019 OM 20150116 Replaced the In-memory compilation with java JAST interpreter.
** 020 OM 20150217 Improved preparePredicate() for OPEN QUERY predicates.
** 021 OM 20150325 Stop processing query if ProgressParser reports syntax error.
** 022 ECF 20150328 Increased query cache size.
** 023 OM 20150307 Improved message printed on fail to process query string.
** 024 OM 20150429 Fixed debug output for the generated (intermediary and final) java code.
** 025 OM 20150518 Refined the logged message when query parsing fails.
** 026 OM 20150617 Compile errors detected during the dynamic code conversion are delegated to
** ErrorManager.
** 027 ECF 20150715 Increased query cache size.
** 028 EVL 20160223 Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
** 029 ECF 20160225 Changes required by PropertyHelper rewrite.
** 030 ECF 20160520 Increased query cache size.
** 031 OM 20160516 Implemented smart caching for queries. Initialize cache size from directory.
** 032 ECF 20160525 Precompute cache key hash code.
** 033 SVL 20160628 Soft handling of dynamic join errors.
** 034 OM 20160905 Small optimizations.
** 035 ECF 20160907 Improved AST string interning for performance.
** 036 SVL 20171024 Some functions are refactored so they can be used by DynamicValidationHelper.
** 037 OM 20181206 Added support for dynamic evaluations.
** 038 OM 20190213 Delayed interpretation of execute() method until the query is open.
** OM 20190321 Specifically evaluate dynamic function calls on query open.
** 039 OM 20190611 Added support for substitution buffers.
** 040 CA 20190812 Changes to allow for mutable buffers; any API which receives a Buffer instance
** and is invoked from converted code must resolve the runtime instance before
** saving the instance.
** OM 20190815 Added substitution buffer support for simple find_ queries.
** ECF 20190827 Fixed SchemaDictionary initialization in the event "default-databases" is
** configured.
** 041 OM 20200109 Added cache support for dynamic queries containing DYNAMIC-FUNCTIONs.
** OM 20200123 Take into consideration constants types when creating level 2 cache key, in
** order to generate different JAST trees.
** 042 CA 20200918 INDEX-INFORMATION requires to interpret the 'execute' method for a dynamic query without
** evaluating the dynamic calls in the WHERE.
** CA 20200930 Use a SymbolResolver exemplar to create the instance used by runtime conversion.
** CA 20210310 A dynamic predicate must set the default lock to NONE, instead of SHARE.
** OM 20210309 Avoid bound exceptions when processing parsing errors.
** ECF 20210914 Fixed schema dictionary temp-table scope caching.
** ME 20211229 Only allow backslash escape character for Unix.
** ECF 20220103 Minimize calls to EnvironmentOps.isUnderWindowsFamily().
** OM 20220112 Fine-tuned error messages for dynamic conversion. Lowered event to WARNING level and
** added ABL source, Java class, method and line number where the event occurred.
** OM 20220224 Added 4GL formatting for errors caused by lexer in addition to those generated by parser.
** OM 20220516 Avoid SIOOBException when the error token is detected outside of the processed string.
** Added new runtime errors caused by invalid ABL code syntax.
** CA 20220614 Conversion errors set the ERROR flag and are allowed to raise a legacy OO exception.
** CA 20221006 Added JMX instrumentation for 'parseFind' and 'parseQuery'. Refs #6814
** OM 20220225 The dynamic queries (and find) have access automatically to all 'sibling' buffers for all
** dataset buffers.
** CA 20220918 Replaced LFUAgingCache with LRUCache.
** OM 20230112 Added finer-granulation instrumentation for processing of dynamic queries.
** SVL 20230113 Improved performance by replacing some "for-each" loops with indexed "for" loops.
** CA 20230116 Avoid using handle.unwrap, handle.getReference or other BDT usage from within FWD runtime.
** 043 OM 20230215 Handled collision of temporary buffers and tables with same name.
** 044 AL2 20230411 Ensure delayed execute is run only once. The flag is reset on query close.
** 045 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 046 EVL 20231124 Avoid duplicating string related methods for performance reason.
** 047 DDF 20240216 lvl1Cache and lvl2Cache can only be created through CacheManager now and
** have been made non-final.
** 048 HC 20240222 Enabled JMX on FWD Client.
** 049 CA 20240305 Dynamic query conversion allows for 'true' = true (poly casting of literals).
** 050 OM 20240318 API changes in DynamicConversionHelper.
** 051 OM 20240327 Improved error handling in case of invalid syntax. Local optimization.
** 052 OM 20240416 Fixed handling of 7328 error condition.
** OM 20240425 Avoid resolution ambiguity by qualifying the table name in simple find queries.
** 053 CA 20240426 Substitution buffers which have the same name as the actual query buffers can not be used
** during the query parse.
** 054 AL2 20240530 Prepare the dictFile as well to be able to work with SchemaWorker in run-time conversion.
** 055 AL2 20240613 Cleared dictFile once the dynamic query is done.
** 056 SP 20240801 Skip using JMX timers when JMX_DEBUG flag is not set.
** 057 AS 20240905 Fixed 7328 error condition not being caught.
** Added support for NumberedException in showConversionError.
** 058 AI 20250220 Added check for JMX_DEBUG to minimize use of lambda.
** 059 RNC 20250318 Do not treat dot or colon as a terminator when inside a comment.
** 060 OM 20250320 Avoided NPE in case an error is generated and the current procedure is unknown.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.persist;
import java.io.*;
import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.*;
import java.util.logging.*;
import antlr.*;
import com.goldencode.ast.*;
import com.goldencode.cache.*;
import com.goldencode.p2j.NumberedException;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.jmx.*;
import com.goldencode.p2j.pattern.*;
import com.goldencode.p2j.persist.P2JQuery.QueryEventListener;
import com.goldencode.p2j.persist.lock.LockType;
import com.goldencode.p2j.schema.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.uast.*;
import com.goldencode.p2j.util.*;
import com.goldencode.p2j.util.ErrorManager;
import com.goldencode.p2j.util.logging.*;
/**
* Helper class for dynamic query conversion.
*/
class DynamicQueryHelper
extends DynamicConversionHelper
implements ProgressParserTokenTypes
{
/** Colon. */
public static final char COLON = ':';
/** Dot. */
public static final char DOT = '.';
/** Single quote. */
public static final char QUOTE = '\'';
/** Double quote. */
public static final char DOUBLE_QUOTE = '"';
/** The backslash character. */
public static final char BACK_SLASH = '\\';
/** Tilde. */
public static final char TILDE = '~';
/** Star. */
public static final char STAR = '*';
/** The forward slash character. */
public static final char FORWARD_SLASH = '/';
/**
* Token to prepend to a find-first predicate in order to obtain a well-formed P4GL query.
* This is also used as a marker to later distinct whether the {@code pcode} is an FIND-FIRST
* or OPEN QUERY statement.
*/
private static final String FIND_FIRST_TOK = "FIND FIRST ";
/** The name of the generated java class. */
private static final String DYN_GEN_QUERY_CLASS = "DynGenQuery";
/** Logger. */
private static final CentralLogger LOG = CentralLogger.get(DynamicQueryHelper.class.getName());
/** A counter of compiled dynamic query classes. */
private static final AtomicLong queryCounter = new AtomicLong(1);
/** Reference to context-local data. */
private static final ContextLocal<WorkArea> local = new ContextLocal<WorkArea>()
{
protected WorkArea initialValue()
{
return new WorkArea();
}
};
/** Instrumentation for {@link #parseFindQuery}. */
private static final NanoTimer PARSE_FIND = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynParseFind);
/** Instrumentation for {@link #parseQuery}. */
private static final NanoTimer PARSE_QUERY = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynParseQuery);
public static final NanoTimer PROCESS_PARSE = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Parse);
public static final NanoTimer PROCESS_PARSE2 = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Parse2);
public static final NanoTimer PROCESS_ANNOTATION = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Annotation);
public static final NanoTimer PROCESS_BASE = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Base);
public static final NanoTimer PROCESS_CORE = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Core);
public static final NanoTimer PROCESS_POST = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Post);
public static final NanoTimer PROCESS_SETUP = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Setup);
public static final NanoTimer PROCESS_SYMRES = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_SymRes);
public static final NanoTimer PROCESS_SCHEMADICT = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_SchemaDict);
public static final NanoTimer PROCESS_INTERN = NanoTimer.getInstance(FwdServerJMX.TimeStat.OrmDynQueryProcess_Intern);
/**
* 1st level cache. Holds the queries and their default parameters. Its management uses
* an aging and usage algorithm to chose entries to be dropped when the maximum capacity
* is reached.
*/
private static ExpiryCache<QueryCacheKey, ParametrizedJast> lvl1Cache;
/**
* 2nd level cache. Queries from this level of cache do not require default parameters
* because they are computed before the cache key. Its management uses an aging and usage
* algorithm to chose entries to be dropped when the maximum capacity is reached.
*/
private static ExpiryCache<QueryCacheKey, JavaAst> lvl2Cache;
/**
* Initialize caches using CacheManager. The default value of the cache
* is used when there is no size available from the configuration.
*/
public static void initializeCache()
{
lvl1Cache = CacheManager.createLRUCache(DynamicQueryHelper.class, "lvl1", 65536);
lvl1Cache.addCacheExpiryListener(event -> {
// debug only: to see when the cache is full
if (LOG.isLoggable(Level.INFO))
{
LOG.log(Level.INFO,
"DynamicQueryHelper cache level #1 is full. " +
"Dumped " + event.getExpiredEntries().size() + " old entries.");
}
});
lvl2Cache = CacheManager.createLRUCache(DynamicQueryHelper.class, "lvl2", 16384);
lvl2Cache.addCacheExpiryListener(event -> {
// debug only: to see when the cache is full
if (LOG.isLoggable(Level.INFO))
{
LOG.log(Level.INFO,
"DynamicQueryHelper cache level #2 is full. " +
"Dumped " + event.getExpiredEntries().size() + " old entries.");
}
});
}
/**
* Private worker for parsing a given query string using the QueryProcessor and return a
* {@link P2JQuery} instance.
* This method is only called by the package-public methods {@link #parseQuery} and
* {@link #parseFindQuery}.
*
* @param processor
* The processor that will generate the Progress code to be converted and later
* post-process the AST before generating the java code.
* @param buffers
* The list of buffers used by the query.
* @param predicate0
* The query predicate.
* @param qname
* The query's name.
* @param substBuffers
* The list of substitution buffers. These are not the main buffers the query is
* iterating but may appear in SUBST nodes (as substitution). By default (when
* {@code null}), each buffer in a dynamic query must appear exactly one time in
* FOR clause.
*
* @return A {@link P2JQuery} instance representing the given legacy query string.
*/
private static P2JQuery parse(QueryProcessor processor,
ArrayList<Buffer> buffers,
String predicate0,
String qname,
List<Buffer> substBuffers)
{
Set<Buffer> allBuffersSet = new LinkedHashSet<>();
Set<String> allBufferNames = new HashSet<>();
for (int j = 0; j < buffers.size(); j++)
{
BufferImpl b = (BufferImpl) buffers.get(j);
allBuffersSet.add(((BufferImpl) b).ref());
allBufferNames.add(b.buffer().getLegacyName().toLowerCase());
// when a dataset buffer is added to a query, all 'sibling' buffers are made available to the query,
// similar to [substBuffers].
if (b.dataSet()._isValid())
{
List<BufferImpl> dsBuffers = ((DataSet) b.dataSet().getResource()).getBuffers();
for (int i = 0; i < dsBuffers.size(); i++)
{
BufferImpl buf = dsBuffers.get(i).ref();
allBuffersSet.add(buf);
allBufferNames.add(buf.buffer().getLegacyName().toLowerCase());
}
}
}
if (substBuffers != null && !substBuffers.isEmpty())
{
for (int i = 0; i < substBuffers.size(); i++)
{
BufferImpl buf = ((BufferImpl) substBuffers.get(i)).ref();
String bufName = buf.buffer().getLegacyName().toLowerCase();
if (!allBufferNames.contains(bufName))
{
allBuffersSet.add(buf);
allBufferNames.add(bufName);
}
}
}
ArrayList<Buffer> allBuffers = new ArrayList<>(allBuffersSet);
JavaAst qAst = null;
ParametrizedJast pJast = null;
final Map<String, Object>[] params = new Map[]{null};
long t0 = System.nanoTime(); // start the timer
String level1Key = getCacheKey(predicate0, null, 1);
QueryCacheKey cache1key = new QueryCacheKey(level1Key, allBuffers);
synchronized (lvl1Cache)
{
// check the cache for a previously constructed JAST
pJast = lvl1Cache.get(cache1key);
}
if (pJast == null)
{
// predicate not found in level#1 cache prepare to search in cache level#2 or even full
// processing in worst case.
final String[] level2Key = {null};
QueryCacheKey cache2key = null;
WorkArea wa = locate();
long queryId = queryCounter.getAndIncrement();
AstManager astManager = AstManager.get();
// set the info about what we are currently parsing
wa.predicate = predicate0;
wa.qname = qname;
wa.buffers = allBuffers;
wa.queryId = queryId;
wa.dynQueryFile = "." + File.separator + "in-mem-p2j-query" + queryId + ".p";
wa.jastFile = wa.dynQueryFile + ".jast";
wa.astFile = wa.dynQueryFile + ".ast";
wa.p2oFile = wa.dynQueryFile + ".p2o";
wa.dictFile = wa.dynQueryFile + ".dict";
try
{
StringBuilder pCode = new StringBuilder();
final SymbolResolver[] sym = new SymbolResolver[1]; // build the symbol resolver
final SchemaDictionary[] dict = new SchemaDictionary[1];
final Map<Buffer, String>[] tempTableNames = new Map[1];
tempTableNames[0] = new HashMap<>();
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_SYMRES.timer(() ->
{
sym[0] = SymbolResolver.newRuntimeInstance();
dict[0] = RecordBuffer.getSchemaDictionary(allBuffers);
sym[0].setSchemaDictionary(dict[0]); // set the schema dictionary used by this context
});
}
else
{
sym[0] = SymbolResolver.newRuntimeInstance();
dict[0] = RecordBuffer.getSchemaDictionary(allBuffers);
sym[0].setSchemaDictionary(dict[0]); // set the schema dictionary used by this context
}
try
{
// load p2o and schema definitions for all temporary buffers
try
{
// load the p2o and schema data
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_SCHEMADICT.timer(
() -> prepareDynamicTables(dict[0],
pCode,
wa.p2oFile,
wa.dictFile,
wa.buffers,
tempTableNames[0]));
}
else
{
prepareDynamicTables(dict[0],
pCode,
wa.p2oFile,
wa.dictFile,
wa.buffers,
tempTableNames[0]);
}
}
catch (PersistenceException exc2)
{
showConversionError((PersistenceException) exc2.getCause(),
"prepare the temp-table buffers for", pCode.toString(), wa.predicate);
return null;
}
catch (RuntimeException exc2)
{
if (exc2.getCause() instanceof PersistenceException)
{
showConversionError(
(PersistenceException) exc2.getCause(),
"prepare the temp-table buffers for", pCode.toString(), wa.predicate);
return null;
}
else
{
throw exc2; // rethrow it
}
}
// add a default scope that represents the scope of the external procedure
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_SYMRES.timer(() -> sym[0].addSchemaScope(false));
}
else
{
sym[0].addSchemaScope(false);
}
// parse the predicate
String predicate = processor.preparePredicate(predicate0);
if (predicate == null)
{
return null;
}
pCode.append(predicate);
boolean complete = false;
ProgressParser[] parser = new ProgressParser[1];
ProgressLexer[] lexer = new ProgressLexer[1];
Exception err = null;
String oeCode = pCode.toString();
StringReader in = new StringReader(oeCode);
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_PARSE.timer(() -> {
lexer[0] = new ProgressLexer(in, sym[0]);
lexer[0].setUnixEscapes(!EnvironmentOps.isUnderWindowsFamily());
parser[0] = new ProgressParser(lexer[0], sym[0]);
});
}
else
{
lexer[0] = new ProgressLexer(in, sym[0]);
lexer[0].setUnixEscapes(!EnvironmentOps.isUnderWindowsFamily());
parser[0] = new ProgressParser(lexer[0], sym[0]);
}
try
{
// throw exception instead of print event to console
parser[0].setConsumeError(false);
// create new parser entry point which supports thw following syntaxes:
// 1. Dynamic find query:
// FIND FIRST <buffer> <find-predicate> <lock-type>.
//
// 2. Dynamic query prepare:
// [ DEFINE BUFFER <buff-name> FOR [TEMP-TABLE] <table-name>. ]*
// OPEN QUERY DynGenQuery <query-predicate>.
// PROCESS_PARSE2.timer(() -> parser[0].external_proc());
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_PARSE2.timer(() -> parser[0].dynamic_query_proc());
}
else
{
parser[0].dynamic_query_proc();
}
complete = true;
}
catch (RecognitionException | TokenStreamException exc2)
{
err = exc2;
}
catch (RuntimeException exc2)
{
if (exc2.getCause() instanceof RecognitionException ||
exc2.getCause() instanceof TokenStreamException)
{
err = exc2;
}
else
{
// if the RecognitionException was re-thrown wrapped as RuntimeException
// from ProgressParser.reportError() we need to get the original back
if (exc2.getCause() instanceof RecognitionException)
{
err = (RecognitionException) exc2.getCause();
}
else
{
// rethrow the Exception of an unknown cause
throw exc2;
}
}
}
if (!complete)
{
showConversionError(err, "parse", oeCode, wa.predicate);
return null;
}
try
{
// prepare the initial AST
ProgressAst finalPAst0 = (ProgressAst) parser[0].getAST();
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_SETUP.timer(() ->
{
finalPAst0.brainwash(wa.dynQueryFile, true);
prepareTree(finalPAst0, wa.buffers, wa.dynQueryFile, DYN_GEN_QUERY_CLASS, tempTableNames[0]);
// NOTE: to disable lvl2 cache and return to old solution, comment the following
// line. If the query is not parametrized, the literals remains hardcoded and
// [params] set to null, so the new JAST will only be stored into lvl1 cache.
params[0] = extractParams(finalPAst0);
level2Key[0] = getCacheKey(predicate, finalPAst0, 2);
});
}
else
{
finalPAst0.brainwash(wa.dynQueryFile, true);
prepareTree(finalPAst0, wa.buffers, wa.dynQueryFile, DYN_GEN_QUERY_CLASS, tempTableNames[0]);
params[0] = extractParams(finalPAst0);
level2Key[0] = getCacheKey(predicate, finalPAst0, 2);
}
// before going forward with conversion, check if we have an already prepared
// JAST for this AST
cache2key = new QueryCacheKey(level2Key[0], allBuffers);
synchronized (lvl2Cache)
{
// check the cache for a previously constructed JAST
qAst = lvl2Cache.get(cache2key);
}
// qAst for this query not found in cache, convert the pAst now
if (qAst == null)
{
// run conversion
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_ANNOTATION.timer(() -> ConversionPool.runTask(ConversionProfile.ANNOTATIONS, finalPAst0));
ProgressAst finalPAst1 = (ProgressAst) astManager.loadTree(wa.astFile);
PROCESS_BASE.timer(() -> ConversionPool.runTask(ConversionProfile.BASE_STRUCTURE, finalPAst1));
ProgressAst finalPAst2 = (ProgressAst) astManager.loadTree(wa.astFile);
PROCESS_CORE.timer(() -> ConversionPool.runTask(ConversionProfile.CORE_CONVERSION, finalPAst2));
}
else
{
ConversionPool.runTask(ConversionProfile.ANNOTATIONS, finalPAst0);
ProgressAst finalPAst1 = (ProgressAst) astManager.loadTree(wa.astFile);
ConversionPool.runTask(ConversionProfile.BASE_STRUCTURE, finalPAst1);
ProgressAst finalPAst2 = (ProgressAst) astManager.loadTree(wa.astFile);
ConversionPool.runTask(ConversionProfile.CORE_CONVERSION, finalPAst2);
}
ProgressAst pAst = (ProgressAst) astManager.loadTree(wa.astFile);
// walk the AST and collect all the buffers
List<String> realBuffers = collectBuffers(pAst);
// validate the buffers
if (!validateBuffers(realBuffers, buffers, wa.qname))
{
return null;
}
JavaAst jCode = (JavaAst) astManager.loadTree(wa.jastFile);
// DBG: System.out.println(jCode.dumpTree(true));
// DBG: System.out.println(ConversionPool.runTask(ConversionProfile.BREW, jCode).getStoredObject(DYN_GEN_QUERY_CLASS));
if (FwdServerJMX.JMX_DEBUG)
{
PROCESS_POST.timer(() -> processor.postprocessJavaAst(jCode));
}
else
{
processor.postprocessJavaAst(jCode);
}
qAst = (JavaAst) astManager.loadTree(wa.jastFile);
if (qAst == null)
{
return null; // conversion failed? this is strange
}
// copy the DynamicEvaluation annotation to JAST to be cached
if (pAst.isAnnotation("DynamicEvaluation")) // if present, it is always true
{
qAst.putAnnotation("DynamicEvaluation", true);
}
// DBG: System.out.println(qAst.dumpTree(true));
// DBG: System.out.println(ConversionPool.runTask(ConversionProfile.BREW, qAst).getStoredObject(DYN_GEN_QUERY_CLASS));
// intern the AST strings for long term storage in both caches
qAst.intern();
if (params[0] != null)
{
// add to level 2 cache only if the query has been parametrised, otherwise
// it will always hit from level 1 cache.
addToCache2(qAst, cache2key);
}
}
else
{
if (LOG.isLoggable(Level.FINE))
{
LOG.log(Level.FINE, "Found in cache#2 " + cache2key + "\n" + qAst.dumpTree());
}
}
// whether the query was found in lvl2 cache or freshly generated we need to
// add it to lvl1 because it was missing from there
addToCache1(qAst, cache1key, params[0]);
}
catch (Exception exc)
{
if (exc.getCause() instanceof CompileException)
{
CompileException cex = (CompileException) exc.getCause();
ErrorManager.recordOrShowError(
cex.getNumber(), cex.getMessage(), true, cex.isPrefix());
}
else
{
showConversionError(exc, "convert", oeCode, wa.predicate);
}
return null;
}
}
finally
{
// clean up the schema
sym[0].deleteSchemaScope(false);
// clean up the progress, java, and P2O ASTs, they are no longer needed
astManager.removeTree(wa.dynQueryFile);
astManager.removeTree(wa.astFile);
astManager.removeTree(wa.jastFile);
astManager.removeTree(wa.p2oFile);
astManager.removeTree(wa.dictFile);
}
}
finally
{
// reset the WorkArea temporary data
wa.predicate = null;
wa.qname = null;
wa.queryId = 0;
wa.buffers = null;
wa.dynQueryFile = null;
wa.jastFile = null;
wa.astFile = null;
wa.p2oFile = null;
wa.dictFile = null;
}
}
else
{
// extract the jast and default parameters
qAst = pJast.jast;
params[0] = pJast.defaultParams;
if (LOG.isLoggable(Level.FINE))
{
LOG.log(Level.FINE, "Found in cache#1 " + cache1key + "\n" + qAst.dumpTree());
}
}
assert (qAst != null);
try
{
RuntimeJastInterpreter interpreter = new RuntimeJastInterpreter(allBuffers, params[0]);
boolean dynamicEvaluation = qAst.isAnnotation("DynamicEvaluation");
interpreter.prepare(qAst, dynamicEvaluation); // load the jast tree
if (!processor.delayedExecute())
{
// interpret the execute() method so that [query0] gets assigned
interpreter.interpret("execute");
}
P2JQuery query0 = (P2JQuery) interpreter.getVariableValue( // extract the variable value
processor.getExpectedQueryName());
if (processor.delayedExecute() && query0 != null)
{
query0.addQueryEventListener(new QueryEventListener() {
private boolean executedOnce = false;
@Override
public void onQueryOpen(boolean evaluate)
{
if (!executedOnce || evaluate)
{
executedOnce = true;
interpreter.processInstancesWith((o) ->
{
if (o instanceof P2JQuery)
{
((P2JQuery) o).setDynamicPredicate(query0.isDynamicPredicate());
}
});
interpreter.interpret("execute");
}
if (evaluate)
{
interpreter.evaluateDynamicCalls();
}
}
@Override
public void onQueryClose()
{
executedOnce = false;
}
});
}
return query0;
}
catch (Throwable t)
{
// Look for ErrorConditionException
Throwable cause;
Throwable error = t;
while ((cause = error.getCause()) != null && (error != cause))
{
error = cause;
if (error instanceof ErrorConditionException)
{
ErrorConditionException e = (ErrorConditionException) error;
ErrorManager.recordOrShowError(e.getProgressErrorCode(), e.getMessage(), false);
return null;
}
}
if (LOG.isLoggable(Level.SEVERE))
{
LOG.log(Level.SEVERE, "Failed to interpret JAST for " + predicate0 + "\n" + qAst.dumpTree(), t);
}
return null;
}
finally
{
if (LOG.isLoggable(Level.FINE))
{
long dt = (System.nanoTime() - t0) / 1000;
LOG.log(Level.FINE, "DQH-Timer = " + dt / 1000.0 + " ms");
}
}
}
/**
* Adds a computed JAST to first level cache along with its default parameters.
* @param jAst
* The JAST to be added.
* @param cacheKey
* The cache key. For level 1 cache, the key should have the default parameters
* @param params
* The default parameters for this query.
*/
private static void addToCache1(JavaAst jAst,
QueryCacheKey cacheKey,
Map<String, Object> params)
{
synchronized (DynamicQueryHelper.lvl1Cache)
{
ParametrizedJast racer = DynamicQueryHelper.lvl1Cache.get(cacheKey);
if (racer == null)
{
// now we can store it for subsequent use
DynamicQueryHelper.lvl1Cache.put(cacheKey, new ParametrizedJast(jAst, params));
if (LOG.isLoggable(Level.FINE))
{
LOG.log(Level.FINE, "Added to cache#1 as " + cacheKey + "\n" + jAst.dumpTree());
}
}
else
{
if (LOG.isLoggable(Level.INFO))
{
LOG.log(Level.INFO, "Query was processed in a parallel thread [" + cacheKey.query + "].");
}
}
}
}
/**
* Adds a computed JAST to second level cache.
*
* @param jAst
* The JAST to be added.
* @param cacheKey
* The cache key. For level 1 cache, the key should have the default parameters
*/
private static void addToCache2(JavaAst jAst, QueryCacheKey cacheKey)
{
synchronized (DynamicQueryHelper.lvl2Cache)
{
JavaAst racer = DynamicQueryHelper.lvl2Cache.get(cacheKey);
if (racer == null)
{
// now we can store it for subsequent use
DynamicQueryHelper.lvl2Cache.put(cacheKey, jAst);
if (LOG.isLoggable(Level.FINE))
{
LOG.log(Level.FINE, "Added to cache#2 as " + cacheKey + "\n" + jAst.dumpTree());
}
}
else
{
if (LOG.isLoggable(Level.INFO))
{
LOG.log(Level.INFO, "Query was processed in a parallel thread [" + cacheKey.query + "].");
}
}
}
}
/**
* Computes a string that will be used in the cache key to uniquely and quickly identify the
* query.
* <p>
* In the case of 1st level cache key, the predicate is used (it should be a trimmed predicate
* with simplified spaces but this is time costly).
* <p>
* For the 2nd level cache, the key is this method returns the UPPERCASED (since the
* {@code predicate} contains no literals) string obtained from the traversal of the
* {@code progAst} in pre-fixed order (as obtained from {@link AnnotatedAst} iterator). To
* reduce the length (simplify) the key string, some of the nodes (that do not add any useful
* information for our purpose) are removed in the process. The result is usually composed from
* the list of the list of injected parameters, followed by the actual query. As noted above,
* the WHERE clause is in prefixed form (operator then operands).
* <p>
* The method returns different key depending on cache key. Usually, the level 1 cache will use
* the predicate for generating a key while 2nd level cache will use the Progress AST as the
* primary source for creating the key.
*
* @param predicate
* The original predicate.
* @param progAst
* The AST tree that was generated on that tree. Not available in level 1 cache.
* @param cacheLevel
* The cache level. In the case that {@code cacheLevel = 1} the {@code progAst} is
* ignored as it is not expected that the Progress AST tree to be built at that time.
*
* @return An string that will identify the predicate in the cache.
*/
private static String getCacheKey(String predicate, ProgressAst progAst, int cacheLevel)
{
if (cacheLevel == 1)
{
return predicate;
// returning a trimmed predicate with simplified spaces is time-costly
// return pred.replaceAll("\\s*", " ").trim();
}
StringBuilder sb = new StringBuilder();
Iterator<Aast> iter = progAst.iterator();
while (iter.hasNext())
{
ProgressAst next = (ProgressAst) iter.next();
switch (next.getType())
{
case ProgressParserTokenTypes.STATEMENT:
case ProgressParserTokenTypes.BLOCK:
case ProgressParserTokenTypes.EXPRESSION:
case ProgressParserTokenTypes.RECORD_PHRASE:
case ProgressParserTokenTypes.DEFINE_VARIABLE:
case ProgressParserTokenTypes.DEFINE_TEMP_TABLE:
case ProgressParserTokenTypes.OPEN_QUERY:
case ProgressParserTokenTypes.QUERY:
case ProgressParserTokenTypes.KW_TEMP_TAB:
case ProgressParserTokenTypes.KW_AS:
continue; // skip useless tokens
}
String text = next.getText();
if (!text.isEmpty())
{
if (sb.length() != 0)
{
sb.append(' ');
}
sb.append(text);
}
}
return sb.toString().toUpperCase();
}
/**
* Parse the given query string and return a {@link P2JQuery} instance.
*
* @param buffers
* The list of buffers used by the query.
* @param predicate
* The query predicate.
* @param qname
* The query's name.
* @param substBuffers
* The list of substitution buffers. These are not the main buffers the query is
* iterating but may appear in SUBST nodes (as substitution). By default (when
* {@code null}), each buffer in a dynamic query must appear exactly one time in
* FOR clause.
*
* @return A {@link P2JQuery} instance representing the given legacy query string.
*/
static P2JQuery parseQuery(ArrayList<Buffer> buffers,
String predicate,
String qname,
List<Buffer> substBuffers)
{
QueryProcessor openQueryProcessor = new QueryProcessor()
{
@Override
public String getExpectedQueryName()
{
return "query0";
}
/**
* Creates a Progress syntax correct {@code OPEN QUERY} statement that uses the
* given predicate. It will be later converted and compiled.
* <p>
* <strong>Note</strong>: This is not a perfect check for query sanity, but it will:
* <ul>
* <li>allow one level nested quotes
* <li>allow handle methods as valid constructs: {@code buffer book:name}
* <li>allow dereference operator (:: punctuation): {@code buffer book::isbn}
* <li>drop garbage after the COLON (:) or DOT (.) when they are terminators
* (followed by some white space)
* </ul>
*
* @param predicate
* The query predicate that needs to be dynamically executed.
*
* @return A string with a Progress query statement that uses the predicate.
*/
@Override
public String preparePredicate(String predicate)
{
int len = predicate.length();
int endIndex = -1;
char quote = '\0';
boolean winOS = EnvironmentOps.isUnderWindowsFamily();
int commentNesting = 0;
for (int i = 0; i < len; i++)
{
char current = predicate.charAt(i);
if (current == QUOTE || current == DOUBLE_QUOTE)
{
if (quote != 0 && quote != current)
{
continue; // nested quotes, not the correct pair!
}
if (i == 0)
{
// actually, will never happen because predicates do not start with
// character literals
quote = current;
}
else
{
char prev = predicate.charAt(i - 1);
if (prev != TILDE && (winOS || prev != BACK_SLASH))
{
// if found the matching quote, mark as not inside the literal
// otherwise (a CHARACTER literal starts) store the current quote type
quote = (quote == current) ? '\0' : current;
}
}
}
else if (quote == '\0')
{
if (i == len - 1)
{
if (current == DOT || current == COLON)
{
// do nothing, current DOT or COLON used as terminator
endIndex = i;
break; // will break anyway
}
}
else
{
char next = predicate.charAt(i + 1);
if (current == DOT || current == COLON)
{
if (commentNesting == 0 &&
(next == ' ' || next == '\t' || next == '\n' || next == '\r'))
{
// current DOT or COLON used as terminator, drop following garbage
endIndex = i;
break;
}
// else the text following DOT or COLON should be the name of a field,
// method or dereference. We don't know at this time, the Progress parser
// must be run to detect syntactic errors.
// Also, the DOT or COLON could be inside a comment,
// so don't treat it as a terminator
}
if (current == FORWARD_SLASH && next == STAR)
{
commentNesting++;
i++;
}
if (current == STAR && next == FORWARD_SLASH)
{
if (commentNesting == 0)
{
// the predicate has invalid syntax. We can stop here since the lexer
// is guaranteed to throw an error
break;
}
commentNesting--;
i++;
}
}
}
}
if (endIndex != -1)
{
// remove garbage if detected
predicate = predicate.substring(0, endIndex);
}
return "OPEN QUERY " + DYN_GEN_QUERY_CLASS + " " + predicate;
}
/**
* Check whether the dynamic query need to delay the interpretation of the
* {@code execute()} method. The {@code execute()} method can be delayed when the
* query variable is assigned when declared as field member in the {@code qAst}. The
* {@code execute()} method must be delayed when the predicate contains a
* {@code dynamic-function} or a normal UDF call.
*
* @return {@code true} when the dynamic query need to delay the interpretation of
* {@code execute()} method.
*/
public boolean delayedExecute()
{
// the queries converted with QUERY-PREPARE are initialized with the declaration so
// they CAN be delayed. When the predicate calls an UDF (no matter if directly or
// dynamically) the [execute] method MUST be delayed until the query is opened
// (via QUERY-OPEN). If no UDF are called in the predicate the result of this method
// is not essential.
return true;
}
/**
* Post-process the intermediary AST to clean it up from unneeded nodes that were
* automatically created during the progress conversion and add accessor for the query.
*
* @param jcode
* The intermediary jast as it was generated form prepared predicate.
*
* @throws AstException
* if any error occurs loading a persisted AST.
*/
@Override
public void postprocessJavaAst(JavaAst jcode)
throws AstException
{
ConversionPool.runTask(ConversionProfile.POSTPROCESS_OPEN_QUERY, jcode);
}
};
P2JQuery res;
if (FwdServerJMX.JMX_DEBUG)
{
res = PARSE_QUERY.timerWithReturn(() -> parse(openQueryProcessor, buffers, predicate, qname, substBuffers));
}
else
{
res = parse(openQueryProcessor, buffers, predicate, qname, substBuffers);
}
return res;
}
/**
* Parse the given predicate string and return a {@link FindQuery} instance.
*
* @param buffer
* The buffer used by the query.
* @param predicate
* The predicate to be parsed.
* @param lock
* The locking mode, may include the NO-WAIT flag.
* @param qname
* The query's name.
* @param substBuffers
* A list of substitution buffers.
*
* @return A {@link P2JQuery} instance representing the given legacy query string.
*/
static P2JQuery parseFindQuery(Buffer buffer,
String predicate,
LockType lock,
String qname,
List<Buffer> substBuffers)
{
WorkArea wa = locate();
try
{
// allow processing unknown predicates
predicate = (predicate == null) ? "" : predicate.trim();
final ArrayList<Buffer> buffers = new ArrayList<>();
buffers.add(buffer);
// save to context local for later use by findQueryProcessor
wa.lock = lock;
QueryProcessor findQueryProcessor = new QueryProcessor()
{
@Override
public String getExpectedQueryName()
{
return "findQuery";
}
/**
* Creates a Progress syntax correct statement that uses the given predicate.
* It will be later converted and compiled. This method uses the context local
* variable lock in order to achieve the required lock / wait behavior.
*
* @param predicate
* Predicate expression for find that evaluates to the following syntax:
* [ WHERE [ logical-expression ] ] [ USE-INDEX index-name ]
*
* @return A string with a Progress {@code FIND FIRST} query statement that
* uses the predicate.
*/
@Override
public String preparePredicate(String predicate)
{
WorkArea wa = locate();
if (wa.buffers == null || wa.buffers.isEmpty())
{
// this should never happen, anyway
return null;
}
// we introduce a FIRST clause that will be later removed in the postprocessing
RecordBuffer rb = ((BufferImpl) wa.buffers.get(0)).buffer();
StringBuilder pCode = new StringBuilder(FIND_FIRST_TOK);
if (!rb.isTemporary())
{
// making sure the buffer is qualified so no ambiguities occurs while parsing the query
pCode.append(rb.getLogicalDatabase()).append(".");
}
pCode.append(rb.getLegacyName()).append(" ").append(predicate);
// then add the required access locking
if (wa.lock.isExclusive())
{
pCode.append(" EXCLUSIVE-LOCK");
}
else if (!wa.lock.isShare())
{
pCode.append(" NO-LOCK");
}
else
{
pCode.append(" SHARE-LOCK"); // default
}
if (wa.lock.isNoWait())
{
pCode.append(" NO-WAIT");
}
pCode.append(".");
return pCode.toString();
}
/**
* Post-process the intermediary AST to clean it up from unneeded nodes that were
* automatically created during the progress conversion and add accessor for
* the query.
*
* @param jcode
* The intermediary jast as it was generated form prepared predicate.
*
* @throws ConfigurationException
* if any error occurs loading the specified configuration profile.
* @throws AstException
* if any error occurs loading a persisted AST.
*/
@Override
public void postprocessJavaAst(JavaAst jcode)
throws ConfigurationException,
AstException
{
ConversionPool.runTask(ConversionProfile.POSTPROCESS_FIND_QUERY, jcode);
}
/**
* Check whether the dynamic query need to delay the interpretation of the
* {@code execute()} method. The {@code execute()} method can be delayed when the
* query variable is assigned when declared as field member in the {@code qAst}. The
* {@code execute()} method must be delayed when the predicate contains a
* {@code dynamic-function} or a normal UDF call.
*
* @return {@code true} when the dynamic query need to delay the interpretation of
* {@code execute()} method.
*/
public boolean delayedExecute()
{
// the FIND-XXXX methods do not support function calls (neither direct nor dynamic)
// also they are the simplest forms and the query is assigned/initialized in
// [execute] method, so it MUST not be delayed.
return false;
}
};
P2JQuery res;
if (FwdServerJMX.JMX_DEBUG)
{
String pred = predicate;
res = PARSE_FIND.timerWithReturn(() -> parse(findQueryProcessor, buffers, pred, qname, substBuffers));
}
else
{
res = parse(findQueryProcessor, buffers, predicate, qname, substBuffers);
}
return res;
}
finally
{
// cleanup
wa.lock = null;
}
}
/**
* Process the {@code progAst} and replace all literals by confectioned variables of the
* required type. The variable names are of a specific pattern so that they don't collide with
* items from converted code and can be easily identified later.
*
* @param progAst
* The Progress AST of a procedure just parsed.
*
* @return A map with newly obtained variables. They are mapped to the {@link BaseDataType}
* values of the literal they replaced. If no literals are found in the process,
* {@code null} is returned.
*/
private static Map<String, Object> extractParams(ProgressAst progAst)
{
// 1st phase: collect literals:
int cnt = 0;
List<ProgressAst> literals = null;
Iterator<Aast> iter = progAst.iterator();
while (iter.hasNext())
{
ProgressAst next = (ProgressAst) iter.next();
switch (next.getType())
{
case ProgressParserTokenTypes.UNKNOWN_VAL:
continue; // skip to the next token
case ProgressParserTokenTypes.KW_NO_LOCK:
case ProgressParserTokenTypes.KW_SH_LOCK:
case ProgressParserTokenTypes.KW_EXC_LOCK:
continue;
case ProgressParserTokenTypes.NUM_LITERAL:
if (next.getAncestor(1).getType() != ProgressParserTokenTypes.LBRACKET)
{
break;
}
int ufo = next.getAncestor(2).getType();
if (ufo > ProgressParserTokenTypes.BEGIN_FIELDTYPES &&
ufo < ProgressParserTokenTypes.END_FIELDTYPES)
{
// this is a FIELD/LBRACKET/EXPRESSION/NUM_LITERAL
// do not replace it with variable because it can be denormalized eventually
continue;
}
}
// the literal nodes are identified by their [is-literal] annotation
// also we add the record LOCK-ing literals
Object ann = next.getAnnotation("is-literal");
if (ann == null || (!(Boolean) ann))
{
continue; // skip to the next token
}
if (cnt == 0)
{
literals = new ArrayList<>();
}
++cnt;
literals.add(next);
}
if (literals == null)
{
return null; // nothing to process
}
// 2nd phase: adjust for poly casting when both operands are literals
Map<String, Object> ret = new HashMap<>(literals.size());
for (int i = 0; i < literals.size(); i++)
{
// specific literal processing for operands: runtime mode allows 'poly' casting of operands; for now,
// only character vs BDT is supported
ProgressAst litAst = literals.get(i);
if (litAst.getType() == STRING)
{
Aast parAst = litAst.getParent();
int indexPos = litAst.getIndexPos();
while (parAst.getType() == LPARENS)
{
if (parAst.getParent().getType() != LPARENS)
{
indexPos = parAst.getIndexPos();
}
parAst = parAst.getParent();
}
// check if operator
int ptype = parAst.getType();
boolean comparison = ptype == EQUALS ||
ptype == NOT_EQ ||
ptype == LT ||
ptype == LTE ||
ptype == GT ||
ptype == GTE;
boolean logical = ptype == KW_AND || ptype == KW_OR || ptype == KW_NOT;
String castFunc = null;
int castType = -1;
int oldtype = -1;
if (logical)
{
castFunc = "logical";
castType = FUNC_LOGICAL;
oldtype = KW_LOGICAL;
}
else if (comparison)
{
// get the other side of the operand
int otherIndex = indexPos == 0 ? 1 : 0;
Aast otherAst = parAst.getChildAt(otherIndex);
while (otherAst.getType() == LPARENS)
{
otherAst = (Aast) otherAst.getFirstChild();
}
if (otherAst.isAnnotation("is-literal") && ((Boolean) otherAst.getAnnotation("is-literal")))
{
int otherType = otherAst.getType();
switch (otherType)
{
case NUM_LITERAL:
castFunc = "int64";
castType = FUNC_INT64;
oldtype = KW_INT64;
break;
case DEC_LITERAL:
castFunc = "decimal";
castType = FUNC_DEC;
oldtype = KW_DEC;
break;
case DATE_LITERAL:
castFunc = "date";
castType = FUNC_DATE;
oldtype = KW_DATE;
break;
case DATETIME_LITERAL:
castFunc = "datetime";
castType = FUNC_DATETIME;
oldtype = KW_DATETIME;
break;
case DATETIME_TZ_LITERAL:
castFunc = "datetime-tz";
castType = FUNC_DATETIME_TZ;
oldtype = KW_DATE_TZ;
break;
case BOOL_TRUE:
case BOOL_FALSE:
castFunc = "logical";
castType = FUNC_LOGICAL;
oldtype = KW_LOGICAL;
break;
default:
castFunc = null;
castType = -1;
oldtype = -1;
break;
}
}
}
if (castFunc != null)
{
// create a builtin function for that type
Aast cast = new ProgressAst();
AbstractConversionWorker.initializeAst(cast,
castType,
castFunc,
litAst.getParent(),
null,
litAst.getIndexPos());
cast.putAnnotation("oldtype", (long) oldtype);
cast.putAnnotation("builtin", true);
cast.putAnnotation("returnsunknown", false);
litAst.move(cast, 0);
}
}
}
// 3nd phase: create with variable of correct type and replace literals
for (int i = 0; i < literals.size(); i++)
{
// Important notes:
// * the new variable name pattern should translate unchanged from P4GL to Java;
// * the pattern should be wisely chosen to exclude possible collisions with identifiers
// from piece of code converted (table/fields, possible variables)
// * variables having this name pattern will be recognized and their definition will be
// removed in postprocess_open_query.xml so that their values will be taken from the
// map returned by this method
String varName = "hqlParam" + (i + 1) + "dq";
Object val = injectVariable(literals.get(i), varName, progAst);
ret.put(varName, val);
}
return ret;
}
/**
* Get the buffer names used by the AST produced for converted predicated.
*
* @param result
* The list of buffer names.
*
* @return See above.
*/
private static ArrayList<String> collectBuffers(ProgressAst result)
{
ArrayList<String> buffers = new ArrayList<>();
Iterator<Aast> iter = result.iterator();
while (iter.hasNext())
{
Aast chAst = iter.next();
if (chAst.getType() != ProgressParserTokenTypes.RECORD_PHRASE)
{
continue;
}
Aast bufAst = (Aast) chAst.getFirstChild();
String bufname = (String) bufAst.getAnnotation("bufname");
int lastDotNdx = bufname.lastIndexOf(".");
if (lastDotNdx != -1)
{
bufname = bufname.substring(lastDotNdx + 1);
}
buffers.add(bufname);
}
return buffers;
}
/**
* Validate the list of buffers against the buffers loaded by this QUERY resource (loaded in
* the {@link WorkArea#buffers} list.
* <p>
* The buffers must have (all conditions must be true):
* <ol>
* <li>The buffers must be in the same order as at the QUERY resource.</li>
* <li>All the buffers loaded by this QUERY resource must be referenced.</li>
* <li>No buffer outside the QUERY's buffer list is allowed.</li>
* </ol>
*
* @param realBuffers
* The list of buffer names used at the query predicated.
* @param iterBuffers
* The list of buffers that need to be iterated by the query. This list does not
* include buffers for substitutions (although the subst list may be included).
* @param queryName
* The query name. Only used to display the error message.
*
* @return {@code true} if the buffers are valid.
*/
private static boolean validateBuffers(List<String> realBuffers,
ArrayList<Buffer> iterBuffers,
String queryName)
{
// check if the expected and converted buffer list match
if (realBuffers.size() != iterBuffers.size())
{
String errMsg =
"QUERY-PREPARE text must have 1 FOR EACH/PRESELECT for each query buffer";
ErrorManager.recordOrShowError(7325, errMsg, true, false, false);
return false;
}
// check buffer positions and if known buffer
for (int i = 0; i < realBuffers.size(); i++)
{
String realBuffer = realBuffers.get(i);
int idx = locateBuffer(iterBuffers, realBuffer);
if (idx == -1)
{
// NOTE: this err message start with a space because the database name that precedes
// it is empty. TODO: Determine when the database name is printed.
String dbName = "";
String errMsg = dbName + " " + realBuffer + " must be an unabbreviated name of a " +
"buffer known in query " + queryName;
ErrorManager.recordOrShowError(7327, errMsg, true, false, false);
return false;
}
if (idx != i)
{
String errMsg = "Buffer " + realBuffer + " is not referenced in the same order in " +
"PREPARE as in BUFFERS list for query " + queryName;
ErrorManager.recordOrShowError(7326, errMsg, true, false, false);
return false;
}
}
return true;
}
/**
* Locate the context-local {@link WorkArea} instance.
*
* @return See above.
*/
private static WorkArea locate()
{
return local.get();
}
/**
* Shows a conversion error in a modal dialog and registers it in the error message list; does
* not set the ERROR-STATUS:ERROR flag. One of the formats of the message is:
* <pre>
* Could not [details] the '{@link WorkArea#predicate}' predicate for query {@link WorkArea#qname}.
* </pre>
* This method does the best effort to match the Progress message by analyzing the failing token.
*
* @param exc
* The exception causing the conversion error.
* @param operation
* The type of operation that failed.
* @param dynCode
* The decorated code being processed (has {@code OPEN QUERY} or {@code FIND FIRST} prefix).
* @param ablCode
* The original progress code being processed, as passed to dynamic procedure.
*/
private static void showConversionError(Exception exc, String operation, String dynCode, String ablCode)
{
String msg = "";
String phrase = null;
WorkArea wa = locate();
boolean isFind = dynCode.startsWith(FIND_FIRST_TOK);
Token tok = null;
if (exc instanceof NoViableAltException)
{
tok = ((NoViableAltException) exc).token;
}
else if (exc instanceof MismatchedTokenException)
{
tok = ((MismatchedTokenException) exc).token;
}
else if (exc instanceof RuntimeException)
{
if (exc.getCause() instanceof MismatchedTokenException)
{
tok = ((MismatchedTokenException) exc.getCause()).token;
}
else if (exc.getCause() instanceof NoViableAltException)
{
tok = ((NoViableAltException) exc.getCause()).token;
}
else if (exc.getCause() instanceof UserGeneratedException)
{
tok = new Token(ProgressParserTokenTypes.SYMBOL, "<unknown>");
}
else if (exc.getCause() instanceof NumberedException)
{
ErrorManager.recordOrShowError(((NumberedException) exc.getCause()).getNumber(),
exc.getMessage(), false);
return;
}
}
else if (exc instanceof TokenStreamRecognitionException)
{
// we do not actually have a token, manufacture one now
TokenStreamRecognitionException tsre = (TokenStreamRecognitionException) exc;
if (tsre.recog instanceof MismatchedCharException)
{
MismatchedCharException recog = (MismatchedCharException) tsre.recog;
if (recog.expecting == '\'' || recog.expecting == '"')
{
int[] errCodes = new int[] {133, 7323};
String[] errMsgs = new String[2];
errMsgs[0] = "** Unmatched quote found in procedure";
errMsgs[1] = "Could not tokenize PREPARE string " + ablCode + "."; // double DOT!
ErrorManager.recordOrShowError(errCodes, errMsgs, true, false, true);
// ** Unmatched quote found in procedure. (133)
// Could not tokenize PREPARE string <predicate-string>. (7323)
return;
}
else if (recog.expecting == '*') // looking for */ <-- closing multiline comment
{
int[] errCodes = new int[] {134, 7323};
String[] errMsgs = new String[2];
errMsgs[0] = "** An open comment string (/*) was found, but no closing string";
errMsgs[1] = "Could not tokenize PREPARE string " + ablCode + "."; // double DOT!
ErrorManager.recordOrShowError(errCodes, errMsgs, true, false, true);
// ** An open comment string (/*) was found, but no closing string. (134)
// Could not tokenize PREPARE string <predicate-string>. (7323)
return;
}
// TODO: are there other unmatched characters the ABL parser handles in a special way?
}
tok = new CommonToken();
tok.setColumn(tsre.recog.getColumn());
tok.setLine(tsre.recog.getLine());
int phraseStart = tok.getColumn() - 13;
if (phraseStart < 0)
{
phraseStart = 0;
}
else
{
while (phraseStart > 0 && dynCode.charAt(phraseStart) != ' ')
{
phraseStart--;
}
}
phrase = dynCode.substring(phraseStart, Math.min(tok.getColumn(), dynCode.length()) - 1)
.replace('\r', ' ').replace('\n', ' ').trim();
}
if (tok != null)
{
if (tok.getType() == ProgressParserTokenTypes.DB_SYMBOL ||
tok.getType() == ProgressParserTokenTypes.SYMBOL)
{
// TODO: try to detect if error 7325: each buffer must be added to dynamic query:
// 'QUERY-PREPARE text must have 1 FOR EACH/PRESELECT for each query buffer'
if (tok.getType() == ProgressParserTokenTypes.SYMBOL)
{
// two spaces are added at the beginning of SYMBOL because the message error is like this:
// <database name> <buffer name> <field name> must be a quoted constant or an unabbreviated,
// unambiguous buffer/field reference for buffers known to query <name>. (7328)
msg = " ";
}
msg += tok.getText() +
" must be a quoted constant or an unabbreviated, unambiguous" +
" buffer/field reference for buffers known to query ";
if (isFind)
{
msg += "or FIND";
}
ErrorManager.recordOrShowError(7328, msg, false, false);
}
else
{
if (phrase == null)
{
int errLoc = tok.getColumn() - 2; // skip a space to previous token
int phraseStart = errLoc - 12; // TODO: this is also heuristic, unknown steps back
while (phraseStart > 0 && dynCode.charAt(phraseStart) != ' ')
{
// make sure we display the entire token
phraseStart--;
}
// find the original predicate substring start
int predicateStart = dynCode.indexOf(wa.predicate);
if (predicateStart < 0)
{
phraseStart = 0; // should never happen because the decorated code is built on wa.predicate
}
// 'drop' back any prefixes we added ro predicate to form a P4GL statement
if (phraseStart < predicateStart)
{
phraseStart = predicateStart;
}
phrase = phraseStart < errLoc && errLoc < dynCode.length()
? dynCode.substring(phraseStart, errLoc).trim()
: "";
if (phrase.isEmpty())
{
// this will happen when the 1st token of the wa.predicate is faulty, because there
// are no tokens before to show, we display it
phrase = tok.getText();
}
}
int[] errCodes = new int[] {247, 0};
String[] errMsgs = new String[2];
errMsgs[0] = msg = ("** Unable to understand after -- \"" + phrase + "\"");
if (isFind)
{
errCodes[1] = 10086;
errMsgs[1] = "FIND METHOD syntax is: [WHERE []] [USE-INDEX ]";
}
else
{
errCodes[1] = 7324;
errMsgs[1] = "PREPARE syntax is: {FOR | PRESELECT} EACH OF.. WHERE ... etc\"";
}
ErrorManager.recordOrShowError(errCodes, errMsgs, true, false, false);
}
}
else
{
msg = "Could not " + operation + " the '" + wa.predicate + "' predicate for query " + wa.qname;
ErrorManager.recordOrShowError(-1, msg, true, false, false);
}
if (LOG.isLoggable(Level.WARNING))
{
// print the important message on WARNING level
ExternalProgramWrapper epw = (ExternalProgramWrapper) ProcedureManager.thisProcedure().getResource();
String ablProcedureName = (epw == null) ? "(memory)" : epw.getFileName().toJavaType();
String javaClass = ProcedureManager.getProcedureHelper()._thisProcedure().getClass().getName();
StackTraceElement[] st = Thread.currentThread().getStackTrace();
StackTraceElement found = null;
for (StackTraceElement stElem : st)
{
if (stElem.getClassName().equals(javaClass))
{
found = stElem;
break;
}
}
String logEntry = (found == null ? javaClass : found.toString()) + " (" + ablProcedureName +
"): failed to dynamically convert P4GL code: '" + dynCode + "'\n" + msg;
if (LOG.isLoggable(Level.FINE))
{
// display the message and a full stack
LOG.log(Level.FINE, logEntry, exc);
}
else
{
// only display the message (contains the legacy procedure, java class, method and line number)
LOG.log(Level.WARNING, logEntry);
}
}
}
/**
* Container of context-local data.
* <p>
* Implements {@link Finalizable} to unregister any surviving dynamic queries from the
* class-loader.
*/
private static class WorkArea
{
// temporary data about the currently converted query
/** The query predicate being converted. */
private String predicate = null;
/** The Id of the current query, automatically incremented. */
long queryId = 0;
/** The locking type for the current query. May be null */
private LockType lock = null;
/** The name of the QUERY resource. */
private String qname = null;
/** The buffers used by the QUERY resource. */
private ArrayList<Buffer> buffers = null;
/** The file name of the class containing the converted query code. */
private String dynQueryFile = null;
/** The name of the generated {@link JavaAst} tree. */
private String jastFile = null;
/** The name of the converted {@link ProgressAst} tree. */
private String astFile = null;
/** The name of the {@link DataModelAst} tree with the temp-table objects for this query. */
private String p2oFile = null;
/** The name of the {@link ProgressAst} tree with the temp-table objects for schema worker. */
private String dictFile = null;
}
/**
* Allows to customize the processing of a predicate for generation of the dynamic query.
*/
private interface QueryProcessor
{
/**
* Returns the name of the variable that will contain the result of the evaluation.
*
* @return the expected variable name.
*/
public String getExpectedQueryName();
/**
* Creates a Progress syntax correct statement that uses the given predicate. It will be
* later converted and compiled.
*
* @param predicate
* The query predicate that needs to be dynamically executed.
*
* @return A string with a Progress query statement that uses the predicate.
*/
public String preparePredicate(String predicate);
/**
* Check whether the dynamic query need to delay the interpretation of {@code execute()}
* method. The {@code execute()} method can be delayed when the query variable is assigned
* when declared as field member in the {@code qAst}. The {@code execute()} method must be
* delayed when the predicate contains a {@code dynamic-function} or a normal UDF call.
*
* @return {@code true} when the dynamic query need to delay the interpretation of
* {@code execute()} method.
*/
public boolean delayedExecute();
/**
* Post-process the intermediary AST to clean it up from unneeded nodes that were
* automatically created during the progress conversion and add accessor for the query.
*
* @param jcode
* The intermediary jast as it was generated form prepared predicate.
*
* @throws ConfigurationException
* if any error occurs loading the specified configuration profile.
* @throws AstException
* if any error occurs loading a persisted AST.
*/
public void postprocessJavaAst(JavaAst jcode)
throws ConfigurationException,
AstException;
}
/**
* Class that keys the already generated JAST trees within the {@code LFUAgingCache} caches of
* {@code DynamicQueryHelper}.
*/
private static class QueryCacheKey
{
/** The query that generated a JAST tree. */
private final String query;
/** The DMO classes of the buffers */
private final Class<?>[] bufferDMOs;
/**
* The names of the buffers
* TODO: Normally this should be removed, or better said, replaced with a buffer matching
* algorithm because there are the same queries, only the name of the buffer differ.
*/
private final String[] bufferNames;
/** Hash code */
private final int hashCode;
/**
* The constructor. Constructs an immutable object that serves as the key for query cache.
*
* @param query
* The query in original 4GL language.
* @param buffers
* The list of {@link Buffer}s the query is based of.
*/
public QueryCacheKey(String query, List<Buffer> buffers)
{
this.query = query;
bufferDMOs = new Class<?>[buffers.size()];
bufferNames = new String[buffers.size()];
int i = 0;
for (Buffer buf : buffers)
{
RecordBuffer rbuf = ((BufferImpl) buf).buffer();
bufferDMOs[i] = rbuf.getDMOImplementationClass();
bufferNames[i] = rbuf.getDMOAlias();
// System.out.println("Buff: name=" + rbuf.getDMOAlias() + ":" + bufferDMOs[i]);
i++;
}
// pre-compute hash code
int result = 17 * query.hashCode();
result = 31 * result + Arrays.hashCode(bufferNames);
hashCode = 31 * result + Arrays.hashCode(bufferDMOs);
}
/**
* Checks whether some other object is "equal to" this one. Both query string and the list
* buffer of buffers must be equals.
*
* @param o
* Another object to compare to.
*
* @return {@code true} if this object is the same as the obj argument.
*/
@Override
public boolean equals(Object o)
{
if (this == o)
{
return true;
}
if (!(o instanceof QueryCacheKey))
{
// objects are only used as keys in the cache so the o will also be of same class
return false;
}
QueryCacheKey that = (QueryCacheKey) o;
return query.equals(that.query) &&
Arrays.equals(bufferDMOs, that.bufferDMOs) &&
Arrays.equals(bufferNames, that.bufferNames);
}
/**
* Returns a pre-computed hash code value for the object. This method is supported for the
* benefit of hash tables such as those provided by {@link java.util.HashMap}.
*
* @return an integer value based on the query string and the buffer involved.
*/
@Override
public int hashCode()
{
return hashCode;
}
/**
* Return a string representation of this object.
*
* @return String representation of this object.
*/
@Override
public String toString() {
return "QueryCacheKey{" +
"query='" + query + '\'' +
", bufferNames=" + Arrays.toString(bufferNames) +
", bufferDMOs=" + Arrays.toString(bufferDMOs) +
'}';
}
}
/**
* A container for holding a JAST and its default parameters. This is used for level 1 caching.
*/
private static class ParametrizedJast
{
/** The JAST that needs to be interpreted. */
private final JavaAst jast;
/**
* The default parameters for rehydrating the parametrized query. This does not take part of
* the actual key, it's just a bundle. It is not null only if the query contains literals
* and it has been parametrized and the key is used in level 1 cache.
* <p>
* After assigning, the parameters must not be altered.
*/
private final Map<String, Object> defaultParams;
/**
* Constructor for this immutable object. Just initializes its members.
*
* @param jast
* The actual JAST to be interpreted.
* @param defaultParams
* The default parameters (if any).
*/
public ParametrizedJast(JavaAst jast, Map<String, Object> defaultParams)
{
this.jast = jast;
this.defaultParams = defaultParams;
}
}
}