ConfigLoader.java

/*
** Module   : ConfigLoader.java
** Abstract : Loads configuration profiles into the pattern engine
**
** Copyright (c) 2005-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
** 001 ECF 20050301   @20145 Created initial version. Parses XML documents
**                           containing pattern engine configuration data
**                           and initializes the pattern engine using this
**                           information.
** 002 ECF 20050303   @20156 Fixed NPE in loadImpl error reporting logic.
** 003 ECF 20050310   @20263 Added support for global includes of named
**                           expression libraries. Previously these were
**                           only possible at the ruleset level.
** 004 ECF 20050310   @20274 Removed language references. This feature set
**                           was obsoleted by a change to the expression
**                           parser which converts language token names
**                           to token types before expression compilation.
** 005 ECF 20050314   @20328 Added user variable support. Loader now reads
**                           and processes a new element 'variable'.
** 006 ECF 20050316   @20412 Added support for ascent/descent rules. These
**                           may be defined for the pattern engine itself
**                           or for any individual ruleset.
** 007 ECF 20050318   @20423 Added support for the action element's "on"
**                           attribute. This attribute determines whether
**                           the action is triggered on true or on false
**                           evaluation of the enclosing rule's condition.
** 008 ECF 20050421   @20812 Import p2j.xml package to use XmlHelper.
** 009 ECF 20050422   @20851 Added support for pattern worker namespace
**                           attribute.
** 010 ECF 20050423   @20868 Allow included rulesets to load pattern
**                           workers.
** 011 GES 20050428   @20918 Added support for next child rules. These
**                           may be defined for the pattern engine itself
**                           or for any individual ruleset.
** 012 ECF 20050430   @20965 Changes related to namespace support for user
**                           functions and variables. Add workers to the
**                           last rule container loaded rather than to the
**                           pattern engine explicitly. This allows
**                           workers to be added to any ruleset. On
**                           include, include entire rule container rather
**                           than only its expression library.
** 013 ECF 20050526   @21299 Changes to support new expression engine
**                           implementation. Change AstSymbolResolver's
**                           rule scope each time lastRuleContainer
**                           changes to ensure any expression processing
**                           which occurs at configuration time is within
**                           the context of the correct scope.
** 014 ECF 20050527   @21326 Allow rules to appear outside of a rule group
**                           block. For instance, a walk rule can now
**                           appear directly beneath the rule-set element.
**                           To do this, the rule must include the "type"
**                           attribute. Valid values for this attribute
**                           are: init, walk, ascent, descent, next-child,
**                           and post.
** 015 GES 20050601   @21361 Add support for parameter lists in named
**                           expressions and handling these as a separate
**                           class.
** 016 GES 20050602   @21379 Added support for arbitrary nesting of 
**                           rule sets.
** 017 GES 20050602   @21389 Added support for a more complete named
**                           function (morphed version of named
**                           expressions).
** 018 GES 20050603   @21398 Added support for nested rules.
** 019 ECF 20050604   @21416 Changes required by new Rule constructor
**                           signature. No longer accepts resolver
**                           parameter.
** 020 GES 20050620   @21522 Changed findConfigurationFile() to be public
**                           and static so it can be used for other files
**                           which we store in the pattern path.
** 021 SIY 20050620   @21239 Added "optional" mode for the rule sets 
**                           loaded from external files.
** 022 GES 20050623   @21546 Added WHILE looping support for implementing
**                           a while element that is backed by a rule. 
** 023 ECF 20050716   @21731 Added maximum walk depth attribute to ruleset
**                           element. Specifies the maximum level to which
**                           the pattern engine iterator should walk an
**                           AST when the target ruleset is applied.
** 024 GES 20050719   @21735 Added support for embedding rules (including
**                           while and all possible nesting and actions)
**                           inside functions.
** 025 ECF 20050726   @21832 Added break and continue support to while
**                           rules. The break and continue XML element can
**                           be added as an action, so long as it is
**                           nested (at any depth) within a while rule.
**                           Both elements support the on true/false
**                           semantic via the "on" attribute.
** 026 GES 20050728   @21898 Enabled support for an honor-hidden attribute for rule-sets.
** 027 GES 20060130   @24157 Honor debug settings before writing output to the terminal.
** 028 ECF 20080721   @39141 Minor optimization. Use StringBuilder instead of StringBuffer.
** 029 GES 20090424   @41933 Import change.
** 030 GES 20090515   @42211 Another import change.
** 031 SVL 20130624          Allow loading conversion resources at server run time.
** 032 CA  20131021          Added conditional rule-set processing.
** 033 HC  20160907          Implemented a simple tracing support for rules processing.
** 034 EVL 20190530          Adding LOG facility to optional rule set processing.
** 035 OM  20200619          Fixes for conditional rule set processing.
**     OM  20220512          Developer mode: allows to load resources from file system if jar is not built.
** 036 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
** 037 OM  20230510          When running the import, use the rules provided with the from jar.
** 038 SP  20240701          Added schema/word_reindex to IMPORT_PROFILES check.
** 039 ICP 20240705          Fixed method of obtaining the class loader.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.pattern;

import java.io.*;
import java.nio.file.*;
import java.util.*;
import java.util.logging.Level;

import com.goldencode.p2j.classloader.MultiClassLoader;
import com.goldencode.p2j.convert.*;
import org.w3c.dom.*;
import com.goldencode.expr.*;
import com.goldencode.util.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.util.*;

/**
 * Loads pattern engine configuration profile data into the pattern engine
 * from one or more XML documents. Top-level pattern engine configuration
 * data is stored in a file with an ".xml" extension. This document may
 * reference zero or more ruleset profiles, each stored in an XML file with
 * a ".rules" extension. Together, a top-level configuration file and the
 * rules profiles it optionally references defines all the information
 * necessary to initialize a pattern engine for a particular application.
 * Please see the <a href="package-summary.html">package summary</a>
 * documentation for a description of these file formats.
 * <p>
 * TODO: this class uses a very unstructured method of reading the DOM, with
 * the assumption that we will eventually validate configuration XML documents
 * against a DTD prior to loading the data into the pattern engine.
 */
public final class ConfigLoader
{
   /**
    * Load condition attribute name. This is taken into consideration only when if conditional rule-sets is
    * activated.
    */
   public static final String ATTR_LC = "load-condition";
   
   /** Execute condition attribute name: execute rule set only if a condition is specified in cfg file. */
   public static final String ATTR_EXECUTE = "execute-condition";
   
   /** Exclude (drop) condition attribute name: ignore rule set if a condition specified in cfg file. */
   public static final String ATTR_DROP = "drop-condition";
   
   /**
    * Optional attribute name: if the rule set is found and can be loaded it is executed normally,
    * otherwise silently ignore the missing file event.
    */
   private static final String ATTR_OPTIONAL = "optional";
   
   /** Key used to obtain the pattern config file search path */
   private static final String PATPATH_KEY = "patpath";
   
   /** File suffix for top-level configuration files */
   private static final String SUFFIX_XML = ".xml";
   
   /** File suffix for ruleset-level configuration files */
   private static final String SUFFIX_RULES = ".rules";
   
   /** XML cfg element tag */
   private static final String ELEM_CFG = "cfg";
   
   /** XML ast-spec element tag */
   private static final String ELEM_ASTSPEC = "ast-spec";
   
   /** XML worker element tag */
   private static final String ELEM_WORKER = "worker";
   
   /** XML variable element tag */
   private static final String ELEM_VARIABLE = "variable";
   
   /** XML parameter element tag */
   private static final String ELEM_PARAMETER = "parameter";
   
   /** XML return element tag */
   private static final String ELEM_RETURN = "return";
   
   /** XML expression element tag */
   private static final String ELEM_EXPRESSION = "expression";
   
   /** XML rule-set element tag */
   private static final String ELEM_RULESET = "rule-set";
   
   /** XML include element tag */
   private static final String ELEM_INCLUDE = "include";
   
   /** XML func-library element tag */
   private static final String ELEM_FUNCLIB = "func-library";
   
   /** XML function element tag */
   private static final String ELEM_FUNCTION = "function";
   
   /** XML init-rules element tag */
   private static final String ELEM_INITRULES = "init-rules";
   
   /** XML walk-rules element tag */
   private static final String ELEM_WALKRULES = "walk-rules";
   
   /** XML ascent-rules element tag */
   private static final String ELEM_ASCENTRULES = "ascent-rules";
   
   /** XML descent-rules element tag */
   private static final String ELEM_DESCENTRULES = "descent-rules";
   
   /** XML next-child-rules element tag */
   private static final String ELEM_NEXTCHILDRULES = "next-child-rules";
   
   /** XML post-rules element tag */
   private static final String ELEM_POSTRULES = "post-rules";
   
   /** XML rule element tag */
   private static final String ELEM_RULE = "rule";
   
   /** XML while element tag */
   private static final String ELEM_WHILE = "while";
   
   /** XML break element tag */
   private static final String ELEM_BREAK = "break";
   
   /** XML continue element tag */
   private static final String ELEM_CONTINUE = "continue";
   
   /** XML action element tag */
   private static final String ELEM_ACTION = "action";
   
   /** XML directory attribute name */
   private static final String ATTR_DIRECTORY = "directory";
   
   /** XML file-spec attribute name */
   private static final String ATTR_FILESPEC = "file-spec";
   
   /** XML input attribute name */
   private static final String ATTR_INPUT = "input";
   
   /** XML depth attribute name */
   private static final String ATTR_DEPTH = "depth";
   
   /** XML name attribute name */
   private static final String ATTR_NAME = "name";
   
   /** Terse mode attribute name */
   private static final String ATTR_OPT = "optional";
   
   /** Honor hidden mode attribute name. */
   private static final String ATTR_HONOR = "honor-hidden";
   
   /** XML access attribute name */
   private static final String ATTR_ACCESS = "access";
   
   /** XML class attribute name */
   private static final String ATTR_CLASS = "class";
   
   /** XML namespace attribute name */
   private static final String ATTR_NAMESPACE = "namespace";
   
   /** XML mode attribute name */
   private static final String ATTR_MODE = "mode";
   
   /** XML type attribute name */
   private static final String ATTR_TYPE = "type";
   
   /** XML on attribute name */
   private static final String ATTR_ON = "on";
   
   /** XML initializer attribute name */
   private static final String ATTR_INIT = "init";
   
   /** Data value constant for tree */
   private static final String VAL_TREE = "tree";
   
   /** Data value constant for view */
   private static final String VAL_VIEW = "view";
   
   /** Data value constant for private */
   private static final String VAL_PRIVATE = "private";
   
   /** Data value constant for public */
   private static final String VAL_PUBLIC = "public";
   
   /** Data value constant for true */
   private static final String VAL_TRUE = "true";
   
   /** Data value constant for false */
   private static final String VAL_FALSE = "false";
   
   /** Location of the conversion-related resources at server run time. */
   private static final String[] CONVERSION_RES_LOCATIONS = new String[] {"rules/", "rules/include/"};
   
   /** The import profiles. */
   private static final Set<String> IMPORT_PROFILES = 
            new HashSet<>(Arrays.asList("schema/import", "schema/word_reindex"));
   
   /** Map of rule type strings to integer constants */
   private static final Map<String, Integer> RULE_TYPES = new HashMap<>();
   
   /** Logger to provide more info for failures. */
   private static final ConversionStatus LOG = ConversionStatus.get(ConfigLoader.class);
   
   /** Flag used for simplify the log by avoiding multiple messages for trying developing mode search. */
   private static boolean developerModeWarned = false;
   
   // Populate rule type map.
   static
   {
      RULE_TYPES.put(null        , RuleContainer.RULE_NONE);
      RULE_TYPES.put("init"      , RuleContainer.RULE_INIT);
      RULE_TYPES.put("walk"      , RuleContainer.RULE_WALK);
      RULE_TYPES.put("ascent"    , RuleContainer.RULE_ASCENT);
      RULE_TYPES.put("descent"   , RuleContainer.RULE_DESCENT);
      RULE_TYPES.put("next-child", RuleContainer.RULE_NEXT_CHILD);
      RULE_TYPES.put("post"      , RuleContainer.RULE_POST);
   }
   
   /** Set of tags which should be processed at the top level. */
   private final Set<String> tags;
   
   /** Pattern engine object being configured. */
   private PatternEngine engine = null;
   
   /** XML element currently being processed. */
   private Element element = null;
   
   /** The originating file name of the currently processed DOM element. */
   private String fileName = null;
   
   /** Rule type for rules last processed. */
   private int ruleType = RuleContainer.RULE_NONE;
   
   /** Rule container last processed. */
   private RuleContainer lastRuleContainer = null;
   
   /** First RuleSet processed. */
   private RuleContainer firstRuleSet = null;
   
   /** Named expression last processed. */
   private NamedFunction lastNamedFunction = null;
   
   /** Rule last added to the ruleset. */
   private Rule lastRule = null;
   
   /** Are we nested within a loop? */
   private boolean inLoop = false;
   
   /**
    * Constructor called by pattern engine; accepts the ruleset object to be
    * initialized. Loading does not begin until the {@link #load} method is
    * called.
    *
    * @param   engine
    *          Pattern engine to be configured with data from an XML
    *          definition.
    */
   public ConfigLoader(PatternEngine engine)
   {
      this(engine, engine, null);
   }
   
   /**
    * Private constructor called to load nested rulesets, for expression
    * library includes, for instance. Processes only the top level elements
    * (i.e., directly under the root <code>rule-set</code> element) whose
    * tag names are included in <code>tags</code>.
    *
    * @param   engine
    *          Pattern engine to be configured with data from an XML
    *          definition.
    * @param   tags
    *          Set of tag names which are to be processed at the top level.
    *          Tag names encountered at the top level in an XML document are
    *          ignored if not included in this set. However, this parameter
    *          may be <code>null</code>, which indicates all top-level tags
    *          are to be processed.
    */
   private ConfigLoader(PatternEngine engine, RuleContainer last, Set<String> tags)
   {
      this.engine = engine;
      this.tags = tags;
      
      setLastRuleContainer(last);
   }
   
   /**
    * Searches the <code>patpath</code> for a file with a root name of
    * <code>name</code> and a suffix of <code>suffix</code>.
    *
    * @param   name
    *          Root of target filename.
    * @param   suffix
    *          Suffix of target filename.
    *
    * @return  The first occurrence of the target file found in the
    *          <code>patpath</code>.
    *
    * @throws  ConfigurationException
    *          if the target file cannot be located.
    */
   public static File findConfigurationFile(String name, String suffix)
   throws ConfigurationException
   {
      String sep = File.pathSeparator;
      String searchPath = Configuration.getParameter(PATPATH_KEY);
      StringBuilder buf = new StringBuilder();
      if (searchPath == null)
      {
         buf.append(".")
            .append(sep);
         searchPath = buf.toString();
         buf.delete(0, buf.length());
      }
      buf.append(name)
         .append(suffix);
      
      String[] paths = searchPath.split(sep);
      try
      {
         return Utils.findFileInPath(buf.toString(), paths);
      }
      catch (FileNotFoundException exc)
      {
         throw new ConfigurationException("Error finding configuration file", exc);
      }
   }
   
   /**
    * Get conversion-related resource at server run time as an input stream.
    *
    * @param  name
    *         Path to the resource, without file suffix. E.g. "schema/hibernate".
    * @param  suffix
    *         File suffix (typically ".xml").
    *
    * @return input stream for reading the target resource or {@code null} if the resource couldn't be found.
    */
   public static InputStream getConversionResource(String name, String suffix)
   {
      for (String location : CONVERSION_RES_LOCATIONS)
      {
         String path = location + name + suffix;
         InputStream stream = MultiClassLoader.getClassLoader().getResourceAsStream(path);
         
         if (stream != null)
         {
            return stream;
         }
      }
      
      // making the best effort of loading the resource from file system to allow developers to get it
      // while working on IDE, without the need to have a full application jar built
      if (!developerModeWarned)
      {
         LOG.log(Level.SEVERE, "Failed to locate '" + name + suffix + "' as a resource. Using developer " +
            "mode: attempting to locate it as local file in project's root directory.");
         developerModeWarned = true;
      }
      
      for (String location : CONVERSION_RES_LOCATIONS)
      {
         String path = location + name + suffix;
         try
         {
            File toFile = Configuration.toFile(path);
            if (toFile.exists())
            {
               return Files.newInputStream(toFile.toPath());
            }
         }
         catch (ConfigurationException | IOException e)
         {
            // ignore current path, skip to next
         }
      }

      LOG.log(Level.SEVERE, "Failed to locate '" + name + suffix + "' resource as a local file.");
      return null; // give up, nowhere to find the file
   }
   
   /**
    * Load the top-level configuration profile from an XML file. The file is
    * named <code>&lt;name&gt;.xml</code>. It must be located somewhere in
    * the path defined by the global configuration parameter
    * <code>patpath</code>.
    *
    * @param   name
    *          Root name of the XML file containing the top-level profile
    *          information.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the configuration data into the
    *          pattern engine.
    */
   public void load(String name)
   throws ConfigurationException
   {
      loadImpl(name, SUFFIX_XML);
   }
   
   /**
    * Load the top-level or ruleset-level configuration profile from an XML
    * file. The file is named <code>&lt;name&gt;.&lt;suffix&gt;</code>. it
    * must be located somewhere in the path defined by the global
    * configuration parameter <code>patpath</code>.
    *
    * @param   name
    *          Root name of the XML file containing the profile information.
    * @param   suffix
    *          Suffix of the name of the XML file containing the profile
    *          information.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the configuration data into the
    *          pattern engine.
    */
   public void loadImpl(String name, String suffix)
   throws ConfigurationException
   {
      try
      {
         Document dom = null;
         if (Configuration.isRuntimeConfig() ||
             Configuration.isImportMode()    ||
             IMPORT_PROFILES.contains(name) && SUFFIX_XML.equals(suffix))
         {
            InputStream stream = getConversionResource(name, suffix);
            if (stream != null)
            {
               dom = XmlHelper.parse(stream);
            }
         }
         
         // developer mode: allow resource to be loaded at runtime from file system if present, avoiding
         // having the jar built
         if (dom == null)
         {
            File file = findConfigurationFile(name, suffix);
            
            if (RulesTracing.isOn)
            {
               Path path = file.toPath().normalize();
               
               // only keep the file name and its directory to limit noise
               fileName = path.subpath(path.getNameCount() - 2, path.getNameCount()).toString();
               
               dom = XmlHelper.parseWithPositions(Files.newInputStream(file.toPath()), false, null, false);
            }
            else
            {
               dom = XmlHelper.parse(file);
            }
         }
         
         if (dom == null)
         {
            throw new ConfigurationException("Cannot find resource: '" + name + suffix + "'.");
         }
         
         element = dom.getDocumentElement();
         if (ELEM_RULESET.equals(element.getTagName()))
         {
            ruleSet();
         }
         else
         {
            processChildElements(true);
         }
      }
      catch (Exception exc)
      {
         StringBuilder buf = new StringBuilder();
         if (element != null)
         {
            Node next = element;
            do
            {
               if (next.getNodeType() == Node.ELEMENT_NODE)
               {
                  Element elem = (Element) next;
                  buf.insert(0, elem.getTagName());
                  buf.insert(0, "/");
               }
               next = next.getParentNode();
            }
            while (next != null);
         }
         else
         {
            buf.append("++ UNAVAILABLE ++");
         }
         
         buf.insert(0, "; XML path: ")
            .insert(0, name)
            .insert(0, "Error loading configuration: ");
         
         throw new ConfigurationException(buf.toString(), exc);
      }
   }
   
   /**
    * Processes the child elements of the element last traversed in the DOM.
    * Dispatches element-specific processing to the appropriate worker method,
    * based on the tag name of the child element encountered.
    * <p>
    * If a limited set of tags is defined for this loader, AND we are not
    * processing an element which represents a rule container, we only
    * process child elements that match an entry in the limited set of tags.
    * This feature is used, for instance, to process only the expression
    * library elements and nested include elements in an included rule
    * container definition.
    *
    * @param   ruleContainerLevel
    *          <code>true</code> if the element currently being processed
    *          represents a rule container, else <code>false</code>.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the configuration data into the
    *          pattern engine.
    */
   private void processChildElements(boolean ruleContainerLevel)
   throws ConfigurationException
   {
      NodeList nodes = element.getChildNodes();
      int len = nodes.getLength();
      for (int i = 0; i < len; i++)
      {
         Node node = nodes.item(i);
         if (node.getNodeType() != Node.ELEMENT_NODE)
         {
            continue;
         }
         
         element = (Element) node;
         String tag = element.getTagName();
         if (!ruleContainerLevel || tags == null || tags.contains(tag))
         {
            if (ELEM_RULE.equals(tag))
            {
               rule(false);
            }
            else if (ELEM_WHILE.equals(tag))
            {
               rule(true);
            }
            else if (ELEM_ACTION.equals(tag))
            {
               action();
            }
            else if (ELEM_BREAK.equals(tag))
            {
               changeLoopAction(true);
            }
            else if (ELEM_CONTINUE.equals(tag))
            {
               changeLoopAction(false);
            }
            else if (ELEM_RULESET.equals(tag))
            {
               ruleSet();
            }
            else if (ELEM_VARIABLE.equals(tag))
            {
               variable(false);
            }
            else if (ELEM_RETURN.equals(tag))
            {
               variable(true);
            }
            else if (ELEM_FUNCTION.equals(tag))
            {
               function();
            }
            else if (ELEM_EXPRESSION.equals(tag))
            {
               expression();
            }
            else if (ELEM_PARAMETER.equals(tag))
            {
               parameter();
            }
            else if (ELEM_INCLUDE.equals(tag))
            {
               include();
            }
            else if (ELEM_FUNCLIB.equals(tag))
            {
               library();
            }
            else if (ELEM_INITRULES.equals(tag))
            {
               initRules();
            }
            else if (ELEM_WALKRULES.equals(tag))
            {
               walkRules();
            }
            else if (ELEM_ASCENTRULES.equals(tag))
            {
               ascentRules();
            }
            else if (ELEM_DESCENTRULES.equals(tag))
            {
               descentRules();
            }
            else if (ELEM_NEXTCHILDRULES.equals(tag))
            {
               nextChildRules();
            }
            else if (ELEM_POSTRULES.equals(tag))
            {
               postRules();
            }
            else if (ELEM_ASTSPEC.equals(tag))
            {
               astSpec();
            }
            else if (ELEM_WORKER.equals(tag))
            {
               worker();
            }
            else
            {
               throw new ConfigurationException(
                  "Tag not recognized: " + tag);
            }
         }
      }
   }
   
   /**
    * Load an AST spec definition from an <code>ast-spec</code> element in a
    * top-level profile definition.
    *
    * @throws  ConfigurationException
    *          if <code>P2J_HOME</code> is not defined for this process.
    */
   private void astSpec()
   throws ConfigurationException
   {
      String dir = element.getAttribute(ATTR_DIRECTORY);
      String spec = element.getAttribute(ATTR_FILESPEC);
      engine.addAstSpec(dir, spec, false);
   }
   
   /**
    * Load a pattern worker helper class definition from a <code>worker</code>
    * element in a top-level profile definition.
    *
    * @throws  ConfigurationException
    *          if any error occurs instantiating or registering the worker
    *          object in the pattern engine.
    */
   private void worker()
   throws ConfigurationException
   {
      String cls = element.getAttribute(ATTR_CLASS);
      String ns = element.getAttribute(ATTR_NAMESPACE);
      ns = StringHelper.safeTrim(ns);
      ns = (ns == null || ns.length() == 0 ? null : ns);
      
      try
      {
         lastRuleContainer.registerWorker(ns, cls);
      }
      catch (Exception exc)
      {
         String msg = "Unable to register worker: " + cls;
         throw new ConfigurationException(msg, exc);
      }
   }
   
   /**
    * Load a user variable definition from a <code>variable</code> element in
    * a top-level or ruleset-level profile definition.
    *
    * @param   ret
    *          <code>true</code> if this is to process a <code>return</code>
    *          value for a named function.
    *
    * @throws  ConfigurationException
    *          if variable's initializer value is invalid, or if
    *          a user variable name collision occurs during registration.
    */
   private void variable(boolean ret)
   throws ConfigurationException
   {
      String name = element.getAttribute(ATTR_NAME);
      String type = element.getAttribute(ATTR_TYPE);
      String value = element.getAttribute(ATTR_INIT);
      
      if (type.trim().length() == 0)
      {
         type = null;
      }
      if (value.trim().length() == 0)
      {
         value = null;
      }
      
      try
      {
         Class<?> dataType = null;
         if (type != null)
         {
            dataType = Class.forName(type);
         }
         
         if (lastNamedFunction != null)
         {
            if (ret)
            {
               // process a return variable for a named function
               lastNamedFunction.registerReturn(name, dataType, value);
            }
            else
            {
               // process a variable for a named function
               lastNamedFunction.registerVariable(name, dataType, value);
            }
         }
         else
         {
            // normal container case
            lastRuleContainer.registerVariable(name, dataType, value);
         }
      }
      catch (Exception exc)
      {
         throw new ConfigurationException(
            "Error initializing user variable " + name, exc);
      }
   }
   
   /**
    * Load a user parameter definition from a <code>parameter</code> element
    * contained in an <code>expression</code> element of a profile
    * definition.
    *
    * @throws  ConfigurationException
    *          if a parameter name collision occurs during registration.
    */
   private void parameter()
   throws ConfigurationException
   {
      String name = element.getAttribute(ATTR_NAME);
      String type = element.getAttribute(ATTR_TYPE);
      
      if (type.trim().length() == 0)
      {
         type = null;
      }
      
      try
      {
         Class<?> dataType = null;
         if (type != null)
         {
            dataType = Class.forName(type);
         }
         
         lastNamedFunction.registerParameter(name, dataType);
      }
      catch (Exception exc)
      {
         throw new ConfigurationException(
            "Error initializing user parameter " + name, exc);
      }
   }
   
   /**
    * Add a user-defined expression to a named function definition.
    */
   private void expression()
   {
      String text = XmlHelper.getText(element);
      lastNamedFunction.registerExpression(text);
   }
   
   /**
    * Load a ruleset definition from a <code>rule-set</code> element in either
    * a top-level or ruleset-level profile definition. This element can either
    * refer to a ruleset definition in an external file (if the
    * <code>name</code> attribute is defined and the element has no child
    * elements), or it can define a ruleset's properties inline using child
    * elements. This method handles both cases.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the ruleset definition from the
    *          current document or from an external document.
    */
   private void ruleSet()
   throws ConfigurationException
   {
      // check conditional rule-set loading; flags are checked only for non-root rule-set nodes
      if (element.getParentNode() != element.getOwnerDocument())
      {
         if (engine.isConditionalRuleSets())
         {
            String loadCondition = element.getAttribute(ATTR_LC).trim();
            
            // fast exit if conditional rule-sets and no flag set
            if (loadCondition.isEmpty())
            {
               return;
            }
            
            Set<String> flags = new TreeSet<>(Arrays.asList(loadCondition.split(",")));
            
            if (!engine.activeFlags(flags))
            {
               return;
            }
         }
         
         if (element.hasAttribute(ATTR_DROP))
         {
            // check the excluding/drop conditions specified in [p2j.cfg.xml]: if any of the tokens in the
            // attribute matches any of the tokens in configuration settings, the rule is dropped
            String dropCondition = element.getAttribute(ATTR_DROP).trim();
            if (!dropCondition.isEmpty())
            {
               Set<String> dropOnList = new TreeSet<>(Arrays.asList(dropCondition.split(",")));
               String parameter = Configuration.getParameter(ATTR_DROP, "");
               if (!parameter.isEmpty())
               {
                  Set<String> cfgExcludeList = new TreeSet<>(Arrays.asList(parameter.split(",")));
                  for (String cond : dropOnList)
                  {
                     if (cfgExcludeList.contains(cond))
                     {
                        return; // drop this rule-set
                     }
                  }
               }
            }
         }
         
         if (element.hasAttribute(ATTR_EXECUTE))
         {
            // check the optional load conditions specified in [p2j.cfg.xml]: none one of the tokens in
            // the attribute matches any tokens in configuration settings, the rule is dropped
            String loadCondition = element.getAttribute(ATTR_EXECUTE).trim();
            if (!loadCondition.isEmpty())
            {
               String parameter = Configuration.getParameter(ATTR_EXECUTE);
               if (!parameter.isEmpty())
               {
                  Set<String> loadOnList = new TreeSet<>(Arrays.asList(loadCondition.split(",")));
                  Set<String> cfgLoadOnList = new TreeSet<>(Arrays.asList(parameter.split(",")));
                  boolean load = false;
                  for (String cond : loadOnList)
                  {
                     if (cfgLoadOnList.contains(cond))
                     {
                        load = true;
                        break;
                     }
                  }
                  
                  if (!load)
                  {
                     return;
                  }
               }
            }
         }
      }
      
      // save off the current container
      RuleContainer old = lastRuleContainer;
      
      int depth = 0;
      String name = element.getAttribute(ATTR_NAME);
      String mode = element.getAttribute(ATTR_INPUT);
      String tmp = element.getAttribute(ATTR_DEPTH);
      if (tmp.trim().length() > 0)
      {
         try
         {
            depth = Integer.parseInt(tmp);
         }
         catch (NumberFormatException exc)
         {
            throw new ConfigurationException("Invalid ruleset depth:  " + tmp);
         }
      }
      
      // determine whether we are in terse mode
      String  optAttr  = element.getAttribute(ATTR_OPT);
      boolean optional = Boolean.TRUE.toString().equals(optAttr);
      
      // determine whether we should honor hidden nodes
      String  honorAttr = element.getAttribute(ATTR_HONOR);
      boolean honor     = Boolean.TRUE.toString().equals(honorAttr);
      
      if (!element.hasChildNodes())
      {
         // This is a reference to an external ruleset.
         try
         {
            ConfigLoader loader = new ConfigLoader(engine, lastRuleContainer, null);
            loader.loadImpl(name, SUFFIX_RULES);
            setLastRuleContainer(loader.lastRuleContainer.addRuleSet());
         }
         catch (ConfigurationException ce)
         {
            if (!optional)
               throw ce;
            
            // optional rule file processing failed, log it
            if (RuleContainer.getDebugLevel() > RuleContainer.MSG_NONE)
            {
               boolean optFileNotFound = false;
               // dig into the causes to find out if the reason was missing rule set itself
               Throwable cause = ce.getCause();
               while(cause != null)
               {
                  if (cause instanceof FileNotFoundException)
                  {
                     optFileNotFound = true;
                     break;
                  }
                  cause = cause.getCause();
               }
               
               // the missing file is not an error, just info about conversion is
               // happening without customer specific files
               if (optFileNotFound)
               {
                  System.out.println("Optional rule set [" + name + "] not found.");
               }
               else
               {
                  LOG.log(Level.SEVERE, "Optional rule set [" + name + "] processing error", ce);
               }
            }
            
            return;
         }
      }
      else
      {
         // This is an inline ruleset.
         setLastRuleContainer(lastRuleContainer.addRuleSet());
      }
      
      // save off the first rule set we encounter
      if (firstRuleSet == null)
      {
         firstRuleSet = lastRuleContainer;
      }
      
      RuleSet ruleSet = (RuleSet) lastRuleContainer;
      ruleSet.setInputMode(VAL_VIEW.equals(mode) ? RuleSet.INPUT_VIEW : RuleSet.INPUT_TREE);
      ruleSet.setDepth(depth);
      ruleSet.setHonorHidden(honor);
      
      processChildElements(true);
      
      // restore the previous container
      setLastRuleContainer(old);
   }
   
   /**
    * Process the <code>include</code> element by including an expression
    * library (and any nested includes) from an external rule container
    * definition.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the included ruleset definition.
    */
   private void include()
   throws ConfigurationException
   {
      Set<String> tags = new HashSet<>();
      tags.add(ELEM_WORKER);
      tags.add(ELEM_INCLUDE);
      tags.add(ELEM_FUNCLIB);
      ConfigLoader loader = new ConfigLoader(engine, lastRuleContainer, tags);
      loader.loadImpl(element.getAttribute(ATTR_NAME), SUFFIX_RULES);
      lastRuleContainer.include(loader.firstRuleSet);
   }
   
   /**
    * Load a named expression library from a <code>library</code> element.
    * If we are currently processing an external document in response to an
    * <code>include</code> element, we only load the library if it is not
    * private.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the library definition.
    *
    * @see     #include
    */
   private void library()
   throws ConfigurationException
   {
      String access = element.getAttribute(ATTR_ACCESS);
      if (VAL_PRIVATE.equals(access) && tags != null)
      {
         return;
      }
      
      processChildElements(false);
   }
   
   /**
    * Load a named function into the ruleset currently being processed from
    * the <code>function</code> element.
    *
    * @throws  ConfigurationException
    *          if any error occurs loading the function definition.
    */
   private void function()
   throws ConfigurationException
   {
      String name = element.getAttribute(ATTR_NAME);
      
      // allow a single expression as the text of the element
      String text = XmlHelper.getText(element);
      if (text != null && text.trim().length() == 0)
      {
         text = null;
      }
      
      lastNamedFunction = new NamedFunction(lastRuleContainer, text);
      lastRuleContainer.putFunction(name, lastNamedFunction);
      
      ruleType = RuleContainer.RULE_WALK;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
      
      lastNamedFunction = null;
   }
   
   /**
    * Prepare to load a set of initialization rules from inside of an
    * <code>init-rules</code> element. This does not actually load the rules,
    * only sets state variables such that when the nested <code>rule</code>
    * tags are encountered, they are processed as init-rules for the correct
    * rule container.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements.
    */
   private void initRules()
   throws ConfigurationException
   {
      ruleType = RuleContainer.RULE_INIT;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
   }
   
   /**
    * Prepare to load a set of AST walking rules from inside of a
    * <code>walk-rules</code> element. This does not actually load the rules,
    * only sets state variables such that when the nested <code>rule</code>
    * tags are encountered, they are processed as walk-rules for the correct
    * rule container.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements.
    */
   private void walkRules()
   throws ConfigurationException
   {
      ruleType = RuleContainer.RULE_WALK;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
   }
   
   /**
    * Prepare to load a list of AST ascent rules from inside of an
    * <code>ascent-rules</code> element. This does not actually load the
    * rules, only sets state variables such that when the nested
    * <code>rule</code> tags are encountered, they are processed as
    * ascent-rules for the correct rule container.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements.
    */
   private void ascentRules()
   throws ConfigurationException
   {
      ruleType = RuleContainer.RULE_ASCENT;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
   }
   
   /**
    * Prepare to load a list of AST descent rules from inside of a
    * <code>descent-rules</code> element. This does not actually load the
    * rules, only sets state variables such that when the nested
    * <code>rule</code> tags are encountered, they are processed as
    * descent-rules for the correct rule container.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements.
    */
   private void descentRules()
   throws ConfigurationException
   {
      ruleType = RuleContainer.RULE_DESCENT;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
   }
   
   /**
    * Prepare to load a list of AST next child rules from inside of an
    * <code>next-child-rules</code> element. This does not actually load the
    * rules, only sets state variables such that when the nested
    * <code>rule</code> tags are encountered, they are processed as
    * next-child-rules for the correct rule container.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements.
    */
   private void nextChildRules()
   throws ConfigurationException
   {
      ruleType = RuleContainer.RULE_NEXT_CHILD;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
   }
   
   /**
    * Prepare to load a set of termination rules from inside of an
    * <code>post-rules</code> element. This does not actually load the rules,
    * only sets state variables such that when the nested <code>rule</code>
    * tags are encountered, they are processed as post-rules for the correct
    * rule container.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements.
    */
   private void postRules()
   throws ConfigurationException
   {
      ruleType = RuleContainer.RULE_POST;
      processChildElements(false);
      ruleType = RuleContainer.RULE_NONE;
   }
   
   /**
    * Load a new rule definition from a <code>rule</code> element. This only
    * creates the rule using its condition expression and remembers the last
    * rule created, to ensure that when nested <code>action</code> tags are
    * encountered, they are processed as actions for this rule.
    *
    * @param   loop
    *          If <code>true</code> this rule should be implemented as a
    *          <code>while</code> loop instead of as an <code>if/else</code>
    *          construct.
    *
    * @throws  ConfigurationException
    *          if any error is encountered processing child elements or
    *          compiling the rule's condition expression.
    */
   private void rule(boolean loop)
   throws ConfigurationException
   {
      int type = ruleType;
      if (type == RuleContainer.RULE_NONE)
      {
         String tmp = element.getAttribute(ATTR_TYPE);
         if (tmp.trim().length() == 0)
         {
            tmp = null;
         }
         Integer wType = RULE_TYPES.get(tmp);
         if (wType != null)
         {
            type = wType;
         }
      }
      
      String expr = XmlHelper.getText(element);
      if (type == RuleContainer.RULE_NONE)
      {
         throw new ConfigurationException("Rule type cannot be determined:  " + expr);
      }
      
      Rule old = null;
      
      if (lastRule != null)
      {
         // save off the current enclosing rule
         old = lastRule;
         
         // nested rules can have the on attribute to define a trigger
         String on = element.getAttribute(ATTR_ON);
         boolean trigger = !VAL_FALSE.equalsIgnoreCase(on);         
         
         // create a new rule and cache it as our current rule being
         // processed
         lastRule = new Rule(expr, type, fileName, getLineNumber());
         
         // add this rule as a nested rule
         old.addRule(lastRule, trigger);
      }
      else
      {
         if (lastNamedFunction != null)
         {
            // this rule is embedded as a "top-level" child of <function>
            lastRule = lastNamedFunction.registerRule(expr, type, fileName, getLineNumber());
         }
         else
         {
            // add this rule to the current rule container, the rule is NOT
            // nested but rather is a "top-level" rule
            lastRule = lastRuleContainer.addRule(expr, type, fileName, getLineNumber());
         }
      }
      
      // at this point, lastRule is the rule that was just created, now we
      // set whether this new rule is a WHILE or an IF/ELSE
      lastRule.setLoop(loop);
      
      boolean oldInLoop = inLoop;
      if (loop)
      {
         inLoop = true;
      }
      
      processChildElements(false);
      
      inLoop = oldInLoop;
      
      // reset our previous last rule or null if this is a top level rule
      lastRule = old;
   }
   
   /**
    * Load an action into the last rule created in response to an
    * <code>action</code> element. The action may be created to trigger on
    * the rule's condition evaluating to <code>true</code> or to
    * <code>false</code>, depending upon the value of the <code>on</code>
    * attribute. The default, if no such attribute is provided, is to
    * trigger on <code>true</code>.
    *
    * @throws  ConfigurationException
    *          if the expression which defines the action cannot be compiled.
    */
   private void action()
   throws ConfigurationException
   {
      String expr = XmlHelper.getText(element);
      String on = element.getAttribute(ATTR_ON);
      boolean trigger = !VAL_FALSE.equalsIgnoreCase(on);
      try
      {
         lastRule.addAction(expr, trigger, fileName,
                            Optional.ofNullable((Integer) element.getUserData("lineNumber"))
                                    .orElse(0));
      }
      catch (ExpressionException exc)
      {
         throw new ConfigurationException("Error compiling action", exc);
      }
   }
   
   /**
    * Load a break or continue action into the last rule created in response
    * to <code>break</code> or <code>continue</code> elements, respectively.
    * The action may be created to trigger on the rule's condition evaluating
    * to <code>true</code> or to <code>false</code>, depending upon the value
    * of the <code>on</code> attribute. The default, if no such attribute is
    * provided, is to trigger on <code>true</code>.
    *
    * @param   isBreak
    *          If <code>true</code>, a break action is added; if
    *          <code>false</code>, a continue action is added.
    *
    * @throws  ConfigurationException
    *          if the expression which defines the action cannot be compiled.
    */
   private void changeLoopAction(boolean isBreak)
   throws ConfigurationException
   {
      String expr = XmlHelper.getText(element);
      String on = element.getAttribute(ATTR_ON);
      boolean trigger = !VAL_FALSE.equalsIgnoreCase(on);
      if (!inLoop)
      {
         throw new ConfigurationException(
            "Tried to add '" + (isBreak ? "break" : "continue") + "' action outside of a loop");
      }
      
      if (isBreak)
      {
         lastRule.addBreak(trigger);
      }
      else
      {
         lastRule.addContinue(trigger);
      }
   }
   
   /**
    * Determines if the element currently being processed is within the scope
    * of the top-level, engine configuration (as opposed to being within the
    * the scope of a lower-level, ruleset configuration.
    *
    * @return  <code>true</code> if our parent element is <code>cfg</code>,
    *          else <code>false</code>.
    */
   private boolean inEngineScope()
   {
      Element parent = (Element) element.getParentNode();
      
      return ELEM_CFG.equals(parent.getTagName());
   }
   
   /**
    * Update the container which will be the target of any configuration
    * to follow, such as registering variables, pattern workers, etc.
    * Also resets the symbol resolver's scope to be this container.
    *
    * @param   container
    *          Rule container which will be the subject of any upcoming
    *          configuration.
    */
   private void setLastRuleContainer(RuleContainer container)
   {
      lastRuleContainer = container;
      AstSymbolResolver.getResolver().setRuleScope(container);
   }

   /**
    * Returns the line number of the currently processed DOM element.
    *
    * @return  See above.
    */
   private int getLineNumber()
   {
      if (RulesTracing.isOn)
      {
         return Optional.ofNullable((Integer) element.getUserData("lineNumber")).orElse(0);
      }
      else
      {
         return 0;
      }
   }
}