# DynamicValidationHelper.java

    /*
    ** Module   : DynamicValidationHelper.java
    ** Abstract : Helper class for dynamic validation expressions.
    **
    ** Copyright (c) 2017-2024, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
    ** 001 SVL 20171024 Created initial version.
    ** 002 ECF 20190827 Fixed SchemaDictionary initialization in the event "default-databases" is configured.
    ** 003 CA  20200930 Use a SymbolResolver exemplar to create the instance used by runtime conversion.
    **     OM  20210309 Do not use DmoMeta as key in TableMapper because temp-tables share the same DmoMeta.
    **     ECF 20210914 Fixed schema dictionary temp-table scope caching.
    **     CA  20220918 Replaced LFUAgingCache with LRUCache.
    **     SVL 20230113 Improved performance by replacing some "for-each" loops with indexed "for" loops.
    **     SVL 20230113 Fixed compilation error.
    **     HC  20230116 Replaced some handle usages with the actual wrapped resources for
    **                  performance.
    **     CA  20230116 Avoid using handle.unwrap, handle.getReference or other BDT usage from within FWD runtime.
    ** 004 OM  20230215 Handled collision of buffers with same dynamic table name.
    ** 005 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    ** 006 DDF 20240216 cache can only be created through CacheManager now and has been made non-final.
    ** 007 OM  20240318 API changes in DynamicConversionHelper.
    ** 008 AL2 20240530 Prepare the dictFile as well to be able to work with SchemaWorker in run-time conversion.
    ** 009 AL2 20240613 Cleared dictFile once the dynamic validation is done.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist;

    import antlr.*;
    import com.goldencode.ast.*;
    import com.goldencode.cache.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.schema.*;
    import com.goldencode.p2j.security.*;
    import com.goldencode.p2j.uast.*;
    import com.goldencode.p2j.ui.*;
    import com.goldencode.p2j.util.ErrorManager;
    import com.goldencode.p2j.util.logging.*;

    import java.io.*;
    import java.util.*;
    import java.util.concurrent.atomic.*;
    import java.util.logging.*;

    /**
     * Helper class for dynamic validation expressions conversion.
     */
    public class DynamicValidationHelper
    extends DynamicConversionHelper
    {
       /** Counter for converted validation expressions. */
       private static final AtomicLong valexpCounter = new AtomicLong(1);

       /** Name of the variable which replaces the validated field reference in the validation expression. */
       public static final String VAL_EXPR_VAR_NAME = "valExprDynParam";

       /** The name of the generated java class. */
       private static final String DYN_GEN_VAL_EXPR_CLASS = "DynGenValExpr";

       /** Logger. */
       private static final CentralLogger LOG = CentralLogger.get(DynamicValidationHelper.class.getName());

       /** Context-local data. */
       private static final ContextLocal<WorkArea> local = new ContextLocal<WorkArea>()
       {
          protected WorkArea initialValue()
          {
             return new WorkArea();
          }
       };

       /**
        * Cache for JASTs generated from validation expressions. Cache management uses an aging and
        * usage algorithm to chose entries to be dropped when the maximum capacity is reached.
        */
       private static ExpiryCache<ValExprCacheKey, JavaAst> cache;

       /**
        * Initializes the cache using CacheManager. The default value of the cache
        * is used when there is no size available from the configuration.
        */
       public static void initializeCache()
       {
          cache = CacheManager.createLRUCache(DynamicValidationHelper.class, 65536);
          cache.addCacheExpiryListener(event ->
          {
             // debug only: to see when the cache is full
             if (LOG.isLoggable(Level.INFO))
             {
                LOG.log(Level.INFO, "DynamicValidationHelper cache is full. Dumped " +
                                    event.getExpiredEntries().size() + " old entries.");
             }
          });
       }

       /**
        * Worker for parsing a given validation expression string and returning a
        * {@link JastValidationExpr} instance.
        *
        * @param  query
        *         Query which iterates the records in the validated buffer.
        * @param  validatedBuffer
        *         Validated buffer.
        * @param  validatedProperty
        *         Property name of the validated field.
        * @param  validateExpression
        *         Validation expression.
        *
        * @return {@link JastValidationExpr} instance representing the given validation expression.
        */
       public static JastValidationExpr parseValidationExpression(QueryWrapper query,
                                                                  RecordBuffer validatedBuffer,
                                                                  String validatedProperty,
                                                                  String validateExpression)
       {
          ValExprCacheKey cacheKey = new ValExprCacheKey(validateExpression, validatedBuffer, validatedProperty);

          int numBuffers = query.numBuffers().intValue();
          ArrayList<Buffer> buffers = new ArrayList<>(numBuffers);
          for (int i = 1; i <= numBuffers; i++)
          {
             buffers.add(query.getBufferByIndex(i));
          }

          JavaAst        javaAst;
          synchronized (cache)
          {
             // check the cache for a previously constructed JAST
             javaAst = cache.get(cacheKey);
          }

          if (javaAst == null)
          {
             long         t0         = System.nanoTime(); // start the timer
             WorkArea     wa         = locate();
             long         valExprId  = valexpCounter.getAndIncrement();
             AstManager   astManager = AstManager.get();

             // set the info about what we are currently parsing
             wa.valExpr      = validateExpression;
             wa.buffers      = buffers;
             wa.dynExprFile = "." + File.separator + "in-mem-valexp" + valExprId + ".p";
             wa.jastFile     = wa.dynExprFile + ".jast";
             wa.astFile      = wa.dynExprFile + ".ast";
             wa.p2oFile      = wa.dynExprFile + ".p2o";
             wa.dictFile     = wa.dynExprFile + ".dict";

             try
             {
                StringBuilder code    = new StringBuilder();
                SymbolResolver sym    = SymbolResolver.newRuntimeInstance(); // build the symbol resolver
                SchemaDictionary dict = RecordBuffer.getSchemaDictionary(buffers);
                sym.setSchemaDictionary(dict); // set the schema dictionary used by this context
                Map<Buffer, String> tempTableNames = new HashMap<>();

                try
                {
                   // load p2o and schema definitions for all temporary buffers

                   try
                   {
                      // load the p2o and schema data
                      prepareDynamicTables(dict, code, wa.p2oFile, wa.dictFile, wa.buffers, tempTableNames);
                   }
                   catch (PersistenceException exc)
                   {
                      displayConversionError(exc, "prepare the temp-table buffers for", code.toString());
                      return null;
                   }

                   // add a default scope that represents the scope of the external procedure
                   sym.addSchemaScope(false);

                   // form converted code
                   code.append("return ");
                   code.append(validateExpression);
                   code.append(".");

                   boolean complete;
                   Exception err         = null;
                   String codeString     = code.toString();
                   StringReader in       = new StringReader(codeString);
                   ProgressLexer lexer   = new ProgressLexer(in, sym);
                   ProgressParser parser = new ProgressParser(lexer, sym);

                   try
                   {
                      // throw exception instead of print event to console
                      parser.setConsumeError(false);
                      complete = parser.external_proc();
                   }
                   catch (RecognitionException | TokenStreamException exc)
                   {
                      err = exc;
                      complete = false;
                   }
                   catch (RuntimeException exc2)
                   {
                      // if the RecognitionException was re-thrown wrapped as RuntimeException
                      // from ProgressParser.reportError() we need to get the original back
                      if (exc2.getCause() instanceof RecognitionException)
                      {
                         err = (RecognitionException) exc2.getCause();
                         complete = false;
                      }
                      else
                      {
                         // rethrow the exception of unknown cause
                         throw exc2;
                      }
                   }

                   if (!complete)
                   {
                      displayConversionError(err, "parse", codeString);
                      return null;
                   }

                   try
                   {
                      ProgressAst pAst = (ProgressAst) parser.getAST();
                      pAst.brainwash(wa.dynExprFile, true);

                      // prepare the initial AST
                      prepareTree(pAst, wa.buffers, wa.dynExprFile, DYN_GEN_VAL_EXPR_CLASS, tempTableNames);

                      TableMapper.LegacyFieldInfo lfi = TableMapper.getLegacyFieldInfo(validatedBuffer,
                                                                                       validatedProperty);
                      String validatedField = lfi.getLegacyName();
                      // get the name of the validate buffer
                      StringBuilder sb = new StringBuilder();
                      if (!validatedBuffer.isTemporary())
                      {
                         sb.append(validatedBuffer.getDatabase().getName()).append(".");
                      }
                      sb.append(validatedBuffer.getLegacyName());
                      String validateBufferName = sb.toString().toLowerCase();

                      replaceParams(pAst, validateBufferName, validatedField);

                      // run conversion
                      ConversionPool.runTask(ConversionProfile.ANNOTATIONS, pAst);
                      pAst = (ProgressAst) astManager.loadTree(wa.astFile);

                      ConversionPool.runTask(ConversionProfile.BASE_STRUCTURE, pAst);
                      pAst = (ProgressAst) astManager.loadTree(wa.astFile);

                      ConversionPool.runTask(ConversionProfile.CORE_CONVERSION, pAst);

                      JavaAst jCode = (JavaAst) astManager.loadTree(wa.jastFile);
                      // DBG: String res1 = jCode.dumpTree(true));
                      // DBG: String res2 = ConversionPool.runTask(
                      //       ConversionProfile.BREW, jCode).getStoredObject(DYN_GEN_VAL_EXPR_CLASS);

                      // TODO using ConversionProfile.POSTPROCESS_OPEN_QUERY may be an overkill,
                      // we can leave only the part necessary for dynamic validation expressions
                      ConversionPool.runTask(ConversionProfile.POSTPROCESS_OPEN_QUERY, jCode);
                      jCode = (JavaAst) astManager.loadTree(wa.jastFile);
                      ConversionPool.runTask(ConversionProfile.POSTPROCESS_VALIDATION_EXP, jCode);
                      javaAst = (JavaAst) astManager.loadTree(wa.jastFile);
                      // DBG: String res3 = ConversionPool.runTask(
                      //       ConversionProfile.BREW, jCode).getStoredObject(DYN_GEN_VAL_EXPR_CLASS);

                      if (javaAst == null)
                      {
                         return null; // conversion failed? this is strange
                      }

                      // intern the AST strings for long term storage in the cache
                      javaAst.intern();

                      addToCache(cacheKey, javaAst);
                   }
                   catch (Exception exc)
                   {
                      if (exc.getCause() instanceof CompileException)
                      {
                         CompileException cex = (CompileException) exc.getCause();
                         ErrorManager.recordOrShowError(
                               cex.getNumber(), cex.getMessage(), true, cex.isPrefix());
                      }
                      else
                      {
                         displayConversionError(exc, "convert", codeString);
                      }
                      return null;
                   }
                }
                finally
                {
                   // clean up the schema
                   sym.deleteSchemaScope(false);

                   // clean up the progress, java, and P2O ASTs, they are no longer needed
                   astManager.removeTree(wa.dynExprFile);
                   astManager.removeTree(wa.astFile);
                   astManager.removeTree(wa.jastFile);
                   astManager.removeTree(wa.p2oFile);
                   astManager.removeTree(wa.dictFile);
                }
             }
             finally
             {
                // reset the WorkArea temporary data
                wa.valExpr = null;
                wa.buffers = null;
                wa.dynExprFile = null;
                wa.jastFile = null;
                wa.astFile = null;
                wa.p2oFile = null;
                wa.dictFile = null;

                if (LOG.isLoggable(Level.FINE))
                {
                   long dt = (System.nanoTime() - t0) / 1000;
                   LOG.log(Level.FINE, "Dynamic-Expression-Timer = " + dt / 1000.0 + " ms");
                }
             }
          }
          else
          {
             if (LOG.isLoggable(Level.FINE))
             {
                LOG.log(Level.FINE, "Found in cache " + cacheKey + "\n" + javaAst.dumpTree());
             }
          }

          return new JastValidationExpr(javaAst, buffers);
       }

       /**
        * Replace field references in the Progress code AST:
        * <ul>
        * <li>references to the validated field are replaced with a variable which is used as a
        * parameter when executing validation expression;</li>
        * <li>reference to buffer's fields other than the currently validated field are replaced with
        * unknown values, warning is shown;</li>
        * <li>reference to fields of other buffers raise error.</li>
        * </ul>
        *
        * @param  progressAst
        *         Progress code AST to be modified.
        * @param  validatedBufferName
        *         Fully qualified legacy name of the validated buffer.
        * @param  validatedField
        *         Legacy name of the validated field.
        */
       private static void replaceParams(ProgressAst progressAst,
                                         String validatedBufferName,
                                         String validatedField)
       {
          String validatedSchemaName = validatedBufferName + "." + validatedField.toLowerCase();

          ArrayList<ProgressAst> unknownNodes = null;
          List<ProgressAst> varNodes = new ArrayList<>();
          Iterator<Aast> iter = progressAst.iterator();
          while (iter.hasNext())
          {
             ProgressAst next = (ProgressAst) iter.next();
             int type = next.getType();
             if (type > ProgressParserTokenTypes.BEGIN_FIELDTYPES &&
                 type < ProgressParserTokenTypes.END_FIELDTYPES)
             {
                String bufname = (String) next.getAnnotation("bufname");
                String schemaname = (String) next.getAnnotation("schemaname");
                if (!bufname.equals(validatedBufferName))
                {
                   // wrapped with RuntimeException in order to be handled with other compile
                   // exceptions in parseValidationExpression
                   throw new RuntimeException(new CompileException(-1,
                      "Progress cannot reference fields other than fields of the validated buffer " +
                      "in a dynamic validation statement. Cannot reference field " + schemaname +
                      ". Validation statement: " + locate().valExpr,
                      next));
                }
                else if (!schemaname.equals(validatedSchemaName))
                {
                   ErrorManager.recordOrShowWarning(-1,
                      "WARNING: in a dynamic validation statement values of the validated buffer's " +
                      "fields other than the currently validated field are UNKNOWN. Replacing " +
                      "reference to " + schemaname + " field with the unknown literal. Validation " +
                      "statement: " + locate().valExpr,
                      false, false, false, false);

                   if (unknownNodes == null)
                   {
                      unknownNodes = new ArrayList<>();
                   }
                   unknownNodes.add(next);
                }
                else
                {
                   varNodes.add(next);
                }
             }
          }

          if (unknownNodes != null)
          {
             for (int i = 0; i < unknownNodes.size(); i++)
             {
                ProgressAst node = unknownNodes.get(i);
                ProgressAst unknown = new ProgressAst();
                unknown.setType(ProgressParserTokenTypes.UNKNOWN_VAL);
                unknown.setText("?");
                node.getParent().graftAt(unknown, node.getIndexPos());
                node.remove();
             }
          }

          if (varNodes.size() > 0)
          {
             injectVariable(varNodes.toArray(new ProgressAst[varNodes.size()]), VAL_EXPR_VAR_NAME, progressAst);
          }
          else
          {
             ErrorManager.recordOrShowWarning(-1,
                "WARNING: validation statement '" + locate().valExpr + "' for " + validatedField +
                " doesn't contain the field itself",
                false, false, false, false);
          }
       }

       /**
        * Add generated JAST to the cache.
        *
        * @param   cacheKey
        *          Cache key.
        * @param   javaAst
        *          Generate JAST.
        */
       private static void addToCache(ValExprCacheKey cacheKey, JavaAst javaAst)
       {
          synchronized (DynamicValidationHelper.cache)
          {
             JavaAst racer = DynamicValidationHelper.cache.get(cacheKey);
             if (racer == null)
             {
                // now we can store it for subsequent use
                DynamicValidationHelper.cache.put(cacheKey, javaAst);

                if (LOG.isLoggable(Level.FINE))
                {
                   LOG.log(Level.FINE, "Added to cache as " + cacheKey + "\n" + javaAst.dumpTree());
                }
             }
             else
             {
                if (LOG.isLoggable(Level.INFO))
                {
                   LOG.log(Level.INFO,
                           "Query was processed in a parallel thread [" + cacheKey.validateExpression + "].");
                }
             }
          }
       }

       /**
        * Display conversion error.
        *
        * @param   exc
        *          Exception caused by the conversion error.
        * @param   operation
        *          Name of the conversion stage at which the error occurred.
        * @param   progressCode
        *          Progress code being converted.
        */
       private static void displayConversionError(Exception exc, String operation, String progressCode)
       {
          WorkArea wa = locate();
          String msg = "Could not " + operation + " the '" + wa.valExpr + "' validation expression";
          ErrorManager.recordOrShowError(-1, msg, true, false, false);

          if (LOG.isLoggable(Level.SEVERE))
          {
             // print the important message on SEVERE level
             LOG.log(Level.SEVERE,
                     "Failed to convert P4GL validation expression code: '" + progressCode + "'\n" +
                     msg);

             if (LOG.isLoggable(Level.FINE))
             {
                // display the full stack on a lower log level
                LOG.log(Level.FINE, "", exc);
             }
          }
       }

       /**
        * Locate the context-local {@link WorkArea} instance.
        *
        * @return   See above.
        */
       private static WorkArea locate()
       {
          return local.get();
       }

       /**
        * Container of context-local data.
        */
       private static class WorkArea
       {
          /** The validation expression being converted. */
          private String valExpr = null;

          /** The buffers used by the validation expression. */
          private ArrayList<Buffer> buffers = null;

          /** The file name of the class containing the converted validation expression code. */
          private String dynExprFile = null;

          /** The name of the generated {@link JavaAst} tree. */
          private String jastFile = null;

          /** The name of the converted {@link ProgressAst} tree. */
          private String astFile = null;

          /**
           * The name of the {@link com.goldencode.p2j.schema.DataModelAst} tree with the temp-table
           * objects for this validation expression.
           */
          private String p2oFile = null;

          /** The name of the {@link ProgressAst} tree with the temp-table objects for schema worker. */
          private String dictFile = null;
       }

       /**
        * Class that keys the JAST tree generated from the validation expression.
        */
       private static class ValExprCacheKey
       {
          /** Validation expression. */
          private final String validateExpression;

          /** DMO class of the validated buffer.  */
          private final Class validatedBufferDMOClass;

          /** Property name of the validate field. */
          private final String validatedProperty;

          /** Hash code. */
          private final int hashCode;

          /**
           * Default constructor.
           *
           * @param   validateExpression
           *          Validation expression.
           * @param   rbuf
           *          Validated record buffer.
           * @param   validatedProperty
           *          Property name of the validate field.
           */
          ValExprCacheKey(String validateExpression, RecordBuffer rbuf, String validatedProperty)
          {
             this.validateExpression = validateExpression;
             this.validatedBufferDMOClass = rbuf.getDMOImplementationClass();
             this.validatedProperty = validatedProperty;

             // pre-compute hash code
             int result = validateExpression.hashCode();
             result = 31 * result + validatedBufferDMOClass.hashCode();
             hashCode = 31 * result + validatedProperty.hashCode();
          }

          /**
           * Checks whether some other object is "equal to" this one.
           *
           * @param   o
           *          Another object to compare to.
           *
           * @return  {@code true} if this object is the same as the obj argument.
           */
          @Override
          public boolean equals(Object o)
          {
             if (this == o)
             {
                return true;
             }

             if (!(o instanceof ValExprCacheKey))
             {
                return false;
             }
             ValExprCacheKey that = (ValExprCacheKey) o;

             return validateExpression.equals(that.validateExpression) &&
                    validatedBufferDMOClass.equals(that.validatedBufferDMOClass) &&
                    validatedProperty.equals(that.validatedProperty);
          }

          /**
           * Get hash code for this cache key.
           *
           * @return hash code for this cache key.
           */
          @Override
          public int hashCode()
          {
             return hashCode;
          }

          /**
           * Return a string representation of this object.
           *
           * @return  string representation of this object.
           */
          @Override
          public String toString()
          {
             return "ValExprCacheKey{" +
                    "validateExpression=" + validateExpression +
                    ", validatedBufferDMOClass=" + validatedBufferDMOClass +
                    ", validatedProperty=" + validatedProperty +
                    "}";
          }
       }
    }