# Variable.java

    /*
    ** Module   : Variable.java
    ** Abstract : contains the data needed to define a Progress variable
    **
    ** Copyright (c) 2005-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ----------------------------------Description----------------------------------
    ** 001 GES 20050210   @19752 First version, simple bean-like approach.
    ** 002 GES 20050330   @20563 Added support for a large number of options
    **                           and helper methods to copy options and write
    **                           them as annotations.
    ** 003 GES 20050401   @20597 Added support for a temporary index to allow
    **                           the linking of a variable reference to its
    **                           original definition at parse time. This is
    **                           needed because project-unique IDs can't be
    **                           easily generated at parse time, but are only
    **                           added after the entire tree is complete.
    ** 004 GES 20050411   @20674 Massive rewrite to use ASTs to store variable
    **                           options and to exchange ASTs with the schema
    **                           dictionary and fully handle field and
    **                           variable options.  Most of the simple getter/
    **                           setter methods are removed since they are
    **                           not generally needed.
    ** 005 GES 20050414   @20724 Cleaned up initializer type conversions and
    **                           added a special booleanManhandle() method
    **                           to post-process a boolean initializer that
    **                           is a user-defined string and thus cannot be
    **                           properly lexed or parsed "in-line".
    ** 006 GES 20050501   @20956 Added annotations of the name and token type
    **                           for the original variable definitions which
    **                           eases processing at conversion time since
    **                           the children don't have to be walked to get
    **                           critical data.
    ** 007 GES 20070329   @32654 Fixed problem with some cases where extent
    **                           options not needed to be converted into a
    **                           non-extent form before the options were
    **                           returned.
    ** 008 GES 20070629   @34347 Minor interface change.
    ** 009 GES 20070919   @35140 Added support for the fully qualified name
    **                           of the class represented by a VAR_CLASS.
    ** 010 GES 20080702   @39057 Number initializers can be strings.
    ** 011 ECF 20080704   @39147 Minor memory optimization. Use Boolean
    **                           constants instead of instantiating new
    **                           Boolean objects.
    ** 012 GES 20090424   @41958 Import change.
    ** 013 GES 20090429   @42068 Match package and class name changes.
    ** 014 GES 20110826          Avoid NPE when there is an unspecified extent.
    **                           Added annotations for 10.2x features.
    ** 015 GES 20111202          Handle boolean initializers for numeric types.
    ** 016 CA  20130209          Added dynamic EXTENT and EXTENT parameter support.
    ** 017 GES 20130307          Added cleanup of literal text before being converted to a
    **                           double. This is needed because the 4GL literal form can be
    **                           incompatible with the Java string form of a double.
    ** 018 VIG 20150108          Changed the method overrideIfNotPresent signature to
    **                           default static.
    ** 019 GES 20150127          Double tildes in format strings from database fields.
    ** 020 CA  20150310          Extracted the logical initialization from string to logical.fromVar.
    ** 021 CA  20150519          Added support for the FONT option at the DEFINE VAR stmt.
    ** 022 EVL 20160224          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
    ** 023 GES 20160309          Fixed booleanManhandle() to allow processing of single quoted
    **                           strings.
    ** 024 GES 20171017          Added hexadecimal literal support.
    ** 025 GES 20181106          Added a builtin flag and record this as an annotation when set.
    **     CA  20181121          Un-indexed .NET-style array variable references must act as
    **                           System.Array.
    ** 026 CA  20181217          Added clsDef - the defining class for this variable, if applied.
    **     GES 20181218          Added annotation of default access modes.
    **     GES 20181228          Added annotation of extent for references (not just the definition).
    ** 027 CA  20190424          Added isStatic.
    ** 028 CA  20190508          Do not set a tempIdx annotation for builtin variables.
    **     CA  20190520          Added isExtent.  Track the legacy signatures for FWD-generated
    **                           methods, to solve converted name collisions with 4GL methods.
    ** 029 CA  20190530          Added support for CONTEXT-HELP-ID widget option.
    ** 030 SVL 20190614          Fix for int64 annotations.
    ** 031 RFB 20191219          Updated annotateOptions to handle INIT creating a determinate EXTENT
    **                           which happens when EXTENT has no parameters, but there is an INIT.
    ** 032 RFB 20200214          Protect annotateClassRef from being called with no class name and
    **                           updated some output messages to be more helpful for class pre-scanning.
    ** 033 CA  20200412          Added incremental conversion support.
    ** 034 CA  20200429          Legacy DEFINE ENUM conversion support.
    ** 035 CA  20200804          Added getExtent().
    ** 036 CA  20210318          Fixed conversion of extent() function/statement with builtin OO properties.
    **     GES 20210618          Track the .NET generic type (if present) and propagate it to references.
    **                           Added a custom mechanism for caller-defined annotations using the
    **                           AnnotationHelper interface.
    **     CA  20211112          XML-NODE-NAME must be annotated at the table field.
    **     TJD 20220504          Java 11 compatibility related minor changes
    **     GES 20220521          Added dataset signature annotation processing.  Cleaned up some warnings.
    **                           Removed unnecessary property handling. Added override flag.
    **     CA  20220531          Add the 'refid' annotation immediately to OO skeleton method calls, as these are
    **                           not included in the post_parse_fixups processing of tempidx.
    **     CA  20220727          OO variable references must get the refid javaname, before checking the javaname
    **                           annotation.
    **     CA  20220727          Improved memory management for parsing; cleanup is done in two phases:
    **                           1. after each legacy class file has finished parsing, the SchemaDictionary will
    **                              keep only protected temp-tables (all private tables are removed, as these
    **                              can't be reached from sub-classes; all permanent tables are removed, as each
    **                              class has its own reference to the permanent tables).  Also, the class def
    **                              instance will reduce its own used memory, which is not required when parsing
    **                              sub-classes.
    **                           2. after parsing of the entire file set is finished, any SchemaDictionary or
    **                              other ASTs referenced by the ClassDefinition are released.
    **     CA  20220728          Fixed CA/20220727 - allow the Variable instance to survive parsing, but release
    **                           the 'def' AST when full parsing finishes.
    **     CA  20220818          An abstract property can define only a getter or a setter, while the implementation
    **                           can define both.  For this reason, only the overridden accessor must have the
    **                           @Override annotation.
    ** 037 CA  20230221          Javadoc fixes.
    ** 038 CA  20230325          For a DEF VAR v2 LIKE v1 EXTENT, if 'v1' has an INIT clause, do not copy it.
    ** 039 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
    ** 040 OM  20230115          Replaced absolutePath(), relativePath(), upPath() and downPath() with faster
    **                           versions, based on node types rather on string paths.
    ** 041 CA  20231004          Added a getter for the 'def' AST node (to be used in read-only mode).
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.uast;

    import com.goldencode.ast.*;
    import com.goldencode.util.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.p2j.convert.*;

    import java.util.logging.*;

    /**
     * A simple wrapper class to contain all related data about a variable. It
     * can also be used to store data related to more simple widgets and frames.
     * All options are stored as an AST and helper methods are provided to set,
     * get, merge and annotate using these option ASTs (which come directly
     * from the source trees or from the schema dictionary).
     * <p>
     * <b>One common option is unusual:</b>
     * <p>
     * <ul>
     *     <li> VIEW-AS phrase (this is a complex subtree, not just a simple
     *          value so it is stored in the AST BUT is cannot be stored into
     *          an annotation at this time)
     * </ul>
     * <p>
     * All options that can be specified in a <code>DEFINE VARIABLE</code>
     * statement are supported except:
     * <p>
     * <ul>
     *     <li> BGCOLOR
     *     <li> CONTEXT-HELP-ID
     *     <li> DCOLOR
     *     <li> DROP-TARGET
     *     <li> FONT
     *     <li> FGCOLOR
     *     <li> MOUSE-POINTER
     *     <li> PFCOLOR
     * </ul>
     * <p>
     * All options that can be specified in <code>DEFINE PARAMETER</code> are
     * supported (this is simply the parameter type itself).
     * <p>
     * All options that can be specified in a <code>format phrase</code> are
     * supported except:
     * <p>
     * <ul>
     *     <li> AUTO-RETURN
     *     <li> BGCOLOR
     *     <li> BLANK
     *     <li> COLON
     *     <li> CONTEXT-HELP-ID
     *     <li> DCOLOR
     *     <li> DEBLANK
     *     <li> DISABLE-AUTO-ZAP
     *     <li> DROP-TARGET
     *     <li> FONT
     *     <li> FGCOLOR
     *     <li> MOUSE-POINTER
     *     <li> NO-TAB-STOP
     *     <li> PFCOLOR
     *     <li> TO
     *     <li> VALIDATE
     * </ul>
     * <p>
     * All options that can be specified in a <code>message</code> statement are
     * supported except <code>AUTO-RETURN</code>.
     *
     * @author    GES
     */
    public class Variable
    implements ProgressParserTokenTypes,
               TokenDataWrapper
    {
       /** Logger. */
       private static final ConversionStatus LOG = ConversionStatus.get(Variable.class);

       /** Possible enclosing quote characters. */
       private final static char[] quotes = { '\"', '\'' };

       /** The full text name of the variable. */
       private String name = null;

       /**
        * Parser defined integer constant matching the token type of the
        * variable.
        */
       private int tokenType = -1;

       /**
        * Temporary index to link variable references with the original AST that
        * is the definition.  This should be set by the creator of the instance
        * before adding this to the dictionary.
        */
       private int tempIdx = -1;

       /** Stores the AST that roots all options associated with the variable. */
       private Aast def = null;

       /** Fully qualified class name of the object instance referenced by this var. */
       private String cls = null;

       /** <code>true</code> if the variable is a built-in global var of the 4GL. */
       private boolean builtin = false;

       /** Flag indicating if this is a .NET array OO definition. */
       private boolean dotNetArray = false;

       /** Optional custom annotation processor. */
       private AnnotationHelper helper = null;

       /** Flag indicating the .NET generic type parameter for this instance, if any ({@code null} means none. */
       private String genericType = null;

       /** Flag to differentiate if the .NET generic type parameter is a primitive type. */
       private boolean genericPrim = false;

       /** Flag to differentiate variables from class properties. */
       private boolean isProp = false;

       /** Flag to differentiate instance and static members. */
       private boolean isStatic = false;

       /** Flag for override option (properties and events only). */
       private boolean isOverride = false;

       /** Flag for abstract option (properties and events only). */
       private boolean isAbstract = false;

       /** Access mode if not defined in a method. */
       private int access = -1;

       /** Extent value (defaults to 0, which is a scalar var). */
       private int extent = 0;

       /** The defining class for this variable, if this is class member. */
       private ClassDefinition clsDef = null;

       /** Flag specifying if this {@link #isProp PROPERTY} has defined a getter accessor. */
       private boolean withGetter = false;

       /** Flag specifying if this {@link #isProp PROPERTY} has defined a setter accessor. */
       private boolean withSetter = false;

       /**
        * Constructs an instance with the required data to be contained.
        *
        * @param    name
        *           The full name of the variable.
        * @param    tokenType
        *           The token type (as defined by the parser) of the variable.
        * @param    cls
        *           Fully qualified class name of the object instance represented
        *           or <code>null</code> if this variable does not represent an
        *           object instance.
        */
       public Variable(String name, int tokenType, String cls)
       throws IllegalArgumentException
       {
          if (name == null || tokenType == -1)
             throw new IllegalArgumentException("Invalid name or type.");

          this.name        = name;
          this.tokenType   = tokenType;
          this.cls         = cls;
       }

       /**
        * Make a copy of the given options, possibly making modifications on the
        * copy.  Specifically, if <code>extent</code> is greater than 0 and the
        * source field is an extent field, the copy will represent a single
        * element of the original extent field.  The effect of this is to remove
        * the KW_EXTENT child node, and to use only the initializer (if any)
        * which corresponds with the element at index <code>extent</code>.
        *
        * @param   source
        *          Options AST to be copied.
        * @param   extent
        *          One-based index of the element (for an extent field) after
        *          which the new field entry is to be modelled. This is only
        *          relevant for the initializer, so that we copy the correct
        *          initial value in the case of an initializer list for an extent
        *          field.  Will be <code>0</code> if the "like" field is not an
        *          extent field, or if the entire extent field is the target of
        *          the like phrase, rather than only a specific element.
        *
        * @return  Copy of the AST corresponding with the given name, with the
        *          possible modifications described above.
        */
       public static Aast copyOptions(Aast source, int extent)
       {
          Aast copy = source.duplicateFresh();

          // Handle special case where like target is a single element of an
          // extent field.
          if (extent > 0)
          {
             // we are changing some values, make sure no conflicting annotations
             // exist
             copy.clearAnnotations();

             // Remove KW_EXTENT child node.
             Aast kwExtent = copy.getImmediateChild(KW_EXTENT, null);
             if (kwExtent != null)
             {
                kwExtent.remove();
             }

             // Normalize index.
             extent--;

             // Remove all KW_INIT children which are not the target we need
             // to retain.
             Aast kwInit = copy.getImmediateChild(KW_INIT, null);
             if (kwInit != null && kwInit.getNumImmediateChildren() > 1)
             {
                // Due to a bug in Progress 9.1c (and possibly other versions),
                // a temp-table field or var like some specific element of any
                // extent field/var always picks up the initial value of the
                // FIRST element in the extent, rather than the correct one.  We
                // emulate this behavior by always setting extent to 0.  The more
                // correct approach is commented out below, in case we need to do
                // it right for other versions.
                Aast initTgt = kwInit.getChildAt(0);
                /*
                ---- BEGIN BETTER APPROACH ----
                Aast initTgt = kwInit.getChildAt(extent);
                if (initTgt == null)
                {
                   int size = kwInit.getNumImmediateChildren();

                   // We have more than one initializer value, but not enough
                   // to match the index we were given, so use the last
                   // available.
                   initTgt = kwInit.getChildAt(size - 1);
                }
                ----- END BETTER APPROACH -----
                */

                // Remove all initializers from the KW_INIT node except the
                // intended target.
                Aast child = kwInit.getChildAt(0);

                while (child != null)
                {
                   Aast next = (Aast) child.getNextSibling();

                   if (child != initTgt)
                   {
                      child.remove();
                   }

                   child = next;
                }
             }
          }

          return copy;
       }

       /**
        * Set the {@link #withGetter} flag.
        *
        * @param    withGetter
        *           <code>true</code> if this {@link #isProp PROPERTY} has a getter defined.
        */
       public void setWithGetter(boolean withGetter)
       {
          this.withGetter = withGetter;
       }

       /**
        * Set the {@link #withSetter} flag.
        *
        * @param    withSetter
        *           <code>true</code> if this {@link #isProp PROPERTY} has a setter defined.
        */
       public void setWithSetter(boolean withSetter)
       {
          this.withSetter = withSetter;
       }

       /**
        * Get the {@link #withGetter} flag.
        *
        * @return   <code>true</code> if this {@link #isProp PROPERTY} has a getter defined.
        */
       public boolean isWithGetter()
       {
          return withGetter;
       }

       /**
        * Get the {@link #withSetter} flag.
        *
        * @return  <code>true</code> if this {@link #isProp PROPERTY} has a setter defined.
        */
       public boolean isWithSetter()
       {
          return withSetter;
       }

       /**
        * Check if this is an extent 4GL variable.
        *
        * @return   See above.
        */
       public boolean isExtent()
       {
          return extent != -2 && extent != 0;
       }

       /**
        * Sets the optional custom annotation processor for this instance.
        *
        * @param    helper
        *           The custom annotation processor.
        */
       public void setAnnotationHelper(AnnotationHelper helper)
       {
          this.helper = helper;
       }

       /**
        * Check if this variable is defined as a .NET array.
        *
        * @return   The {@link #dotNetArray} state.
        */
       public boolean isDotNetArray()
       {
          return dotNetArray;
       }

       /**
        * Set the flag for .NET array.
        *
        * @param    dotNetArray
        *           The new value of the flag.
        */
       public void setDotNetArray(boolean dotNetArray)
       {
          this.dotNetArray = dotNetArray;
       }

       /**
        * Get .NET generic type parameter, if it exists.
        *
        * @return   The {@link #genericType} state.
        */
       public String getGenericType()
       {
          return genericType;
       }

       /**
        * Set the .NET generic type parameter.
        *
        * @param    genericType
        *           The .NET generic type parameter.
        */
       public void setGenericType(String genericType)
       {
          this.genericType = genericType;
       }

       /**
        * Returns the state of the flag to mark the generic type parameter as a
        * primitive.
        *
        * @return   <code>true</code> if this .NET generic type is primitive.
        */
       public boolean isGenericPrimitive()
       {
          return genericPrim;
       }

       /**
        * Sets the flag to mark the generic type parameter as a primitive.
        *
        * @param    prim
        *           <code>true</code> if this .NET generic type is primitive.
        */
       public void setGenericPrimitive(boolean prim)
       {
          this.genericPrim = prim;
       }

       /**
        * Returns the full name of the variable.
        *
        * @return   The full name of the variable.
        */
       public String getName()
       {
          return name;
       }

       /**
        * Sets the full name of the variable.
        *
        * @param    name
        *           The full name of the variable.
        */
       public void setName(String name)
       {
          this.name = name;
       }

       /**
        * Returns the parser-defined token type of the variable.
        *
        * @return   The token type of the variable.
        */
       public int getTokenType()
       {
          return tokenType;
       }

       /**
        * Sets the parser-defined token type of the variable.
        *
        * @param    tokenType
        *           The token type of the variable.
        */
       public void setTokenType(int tokenType)
       {
          this.tokenType = tokenType;
       }

       /**
        * Returns the state of the built-in global variable flag.
        *
        * @return   <code>true</code> if this is a built-in global variable.
        */
       public boolean isBuiltin()
       {
          return builtin;
       }

       /**
        * Sets the state of the built-in global variable flag.
        *
        * @param    builtin
        *           <code>true</code> if this is a built-in global variable.
        */
       public void setBuiltin(boolean builtin)
       {
          this.builtin = builtin;
       }

       /**
        * Returns the property flag.
        *
        * @return   <code>true</code> if this is a property.
        */
       public boolean isProp()
       {
          return isProp;
       }

       /**
        * Returns the override flag.
        *
        * @return   <code>true</code> if this is an overridden property or event.
        */
       public boolean isOverride()
       {
          return isOverride;
       }

       /**
        * Returns the abstract flag.
        *
        * @return   <code>true</code> if this is an abstract property or event.
        */
       public boolean isAbstract()
       {
          return isAbstract;
       }

       /**
        * Returns the static flag.
        *
        * @return   <code>true</code> if this is a static member.
        */
       public boolean isStatic()
       {
          return isStatic;
       }

       /**
        * Sets the state of the override flag.
        *
        * @param    isOverride
        *           <code>true</code> if this is an overridden property or event.
        */
       public void setOverride(boolean isOverride)
       {
          this.isOverride = isOverride;
       }

       /**
        * Sets the state of the abstract flag.
        *
        * @param    isAbstract
        *           <code>true</code> if this is an abstract property or event.
        */
       public void setAbstract(boolean isAbstract)
       {
          this.isAbstract = isAbstract;
       }

       /**
        * Sets the state of the property flag.
        *
        * @param    isProp
        *           <code>true</code> if this is a property.
        */
       public void setProperty(boolean isProp)
       {
          this.isProp = isProp;
       }

       /**
        * Sets the state of the static flag.
        *
        * @param    isStatic
        *           <code>true</code> if this is a static member.
        */
       public void setStatic(boolean isStatic)
       {
          this.isStatic = isStatic;
       }

       /**
        * Set the {@link #access} mode.
        *
        * @param    access
        *           The access mode.
        */
       public void setAccess(int access)
       {
          this.access = access;
       }

       /**
        * Set the {@link #extent} value.
        *
        * @param    extent
        *           The extent value.
        */
       public void setExtent(int extent)
       {
          this.extent = extent;
       }

       /**
        * Get the {@link #extent} value.
        *
        * @return   See above.
        */
       public int getExtent()
       {
          return extent;
       }

       /**
        * Returns the temporary index that identifies the original AST node that
        * defined the variable.
        *
        * @return   The temporary index.
        */
       public int getTempIndex()
       {
          return tempIdx;
       }

       /**
        * Sets the temporary index that identifies the original AST node that
        * defined the variable.
        *
        * @param    tempIdx
        *           The temporary index.
        */
       public void setTempIndex(int tempIdx)
       {
          this.tempIdx = tempIdx;
       }

       /**
        * Obtains the fully qualified name of the class of the object instance
        * represented by this variable.
        *
        * @return   The fully qualified name of the class of the object instance
        *           represented by this variable or <code>null</code> if this
        *           variable does not represent an object instance.
        */
       public String getClassName()
       {
          return cls;
       }

       /**
        * Sets the fully qualified name of the class of the object instance
        * represented by this variable.
        *
        * @param    cls
        *           The fully qualified name of the class of the object instance
        *           represented by this variable or <code>null</code> if this
        *           variable does not represent an object instance.
        */
       public void setClassName(String cls)
       {
          this.cls = cls;
       }

       /**
        * Get the defining class of this variable, if this is a class member.
        *
        * @return   The {@link #clsDef}.
        */
       public ClassDefinition getClassDefinition()
       {
          return clsDef;
       }

       /**
        * Set the defining class of this variable.
        *
        * @param    clsDef
        *           The defining class.
        */
       public void setClassDefinition(ClassDefinition clsDef)
       {
          this.clsDef = clsDef;
       }

       /**
        * Copies all options based on another object of this same type.  The name,
        * token type and temporary index are NOT overridden.
        *
        * @param    like
        *           The instance from which to copy the options.
        */
       public void copyDefaultOptions(Aast like)
       {
          if (like != null)
          {
             def = like.duplicate(null);
          }
       }

       /**
        * Sets all options based on the children of the passed-in AST, overriding
        * any options previously copied based on a <code>LIKE</code> clause.  The
        * name, token type and temporary index are NOT overridden.  <b>This merge
        * process may create new AST nodes where none existed in the original
        * source tree!</b>
        * <p>
        * The following options are handled:
        * <p>
        * <ul>
        *    <li> FORMAT
        *    <li> INITIAL
        *    <li> LABEL
        *    <li> COLUMN-LABEL
        *    <li> DECIMALS
        *    <li> EXTENT
        *    <li> CASE-SENSITIVE
        *    <li> NOT CASE-SENSITIVE
        *    <li> VIEW-AS
        * </ul>
        *
        * @param    ast
        *           The AST whose children define the options to be set.
        */
       public void setOptions(Aast ast)
       {
          // are there any previous options to consider?
          if (def == null)
          {
             // simple case: no previous like clause
             def = ast;
             return;
          }

          // merge old options into AST
          overrideIfNotPresent(def, ast, KW_FORMAT);
          if (!(def.downPath(KW_INIT) && ast.downPath(KW_EXTENT) && !ast.downPath(KW_EXTENT, NUM_LITERAL)))
          {
             // do not copy INIT option if this is a DEF VAR ... LIKE ... EXTENT.
             overrideIfNotPresent(def, ast, KW_INIT);
          }
          overrideIfNotPresent(def, ast, KW_LABEL);
          overrideIfNotPresent(def, ast, KW_COL_LAB);
          overrideIfNotPresent(def, ast, KW_DECIMALS);
          overrideIfNotPresent(def, ast, KW_EXTENT);
          overrideIfNotPresent(def, ast, KW_VIEW_AS);
          overrideIfNotPresent(def, ast, KW_SERIALZN);

          // special processing for case-sensitive (only copy if neither option
          // exists in the target AST)
          if (ast.getImmediateChild(KW_NOT, null)      == null &&
              ast.getImmediateChild(KW_CASE_SEN, null) == null)
          {
             // try copying the NOT CASE-SENSITIVE
             if (overrideIfNotPresent(def, ast, KW_NOT) == false)
             {
                // try copying the CASE-SENSITIVE
                overrideIfNotPresent(def, ast, KW_CASE_SEN);
             }
          }

          // save off the merged AST
          def = ast;
       }

       /**
        * Gets the AST that defines all options for this variable. These options
        * may have been copied based on a <code>LIKE</code> clause or set via a
        * source code definition.  The name, token type and temporary index are
        * NOT accessible via the returned AST.
        * <p>
        * The returned options may have been modified. Specifically, if
        * <code>extent</code> is greater than 0 and the source variable is an
        * extent variable, the result will represent a copy of a single element
        * of the original extent variable. The effect of this is to remove the
        * KW_EXTENT child node, and to use only the initializer (if any) which
        * corresponds with the element at index <code>extent</code>.
        *
        * @param   extent
        *          One-based index of the element (for an extent field) after
        *          which the new field entry is to be modelled. This is only
        *          relevant for the initializer, so that we copy the correct
        *          initial value in the case of an initializer list for an extent
        *          field.  Will be <code>0</code> if the "like" field is not an
        *          extent field, or if the entire extent field is the target of
        *          the like phrase, rather than only a specific element.
        *
        * @return   The AST whose children define the options for this variable.
        */
       public Aast getOptions(int extent)
       {
          int  len      = 0;
          Aast possible = def.getImmediateChild(KW_EXTENT, null);

          if (possible != null)
          {
             len = -1; // dynamic-extent

             if (possible.downPath(NUM_LITERAL))
             {
                len = Integer.parseInt(possible.getFirstChild().getText());
             }
             else if (possible.downPath(HEX_LITERAL))
             {
                String hex = possible.getFirstChild().getText();
                decimal d = decimal.fromUnsignedHexLiteral(hex);
                len = d.intValue();
             }
          }

          // WARNING: should we only return a copy here? It doesn't seem to
          //          cause any trouble, but it seems dangerous. We could avoid
          //          all of this work and just let copyOptions() do the job...
          if (extent == 0 || possible == null || len == 0)
             return def;

          return copyOptions(def, extent);
       }

       /**
        * Stores all non-standard options as annotations to the AST passed as a parameter if this is the
        * original definition, otherwise it stores a subset of values, most important of which is the cross-
        * reference index to allow the original definition to be easily identified and accessed from a
        * reference.  If an option has the default value, it is not saved as an annotation.
        * <p>
        * At the end of the processing, if a custom annotation helper has been defined for this instance, it
        * will be invoked.
        *
        * @param    ast
        *           The AST to annotate.
        * @param    original
        *           <code>true</code> if this is to annotate the AST of the original definition,
        *           <code>false</code> if the AST is only a reference.
        * @param    inMethod
        *           <code>true</code> if this is being done from within a method definition.
        * @param    sym
        *           The current symbol resolver.
        */
       public void annotateOptions(Aast ast, boolean original, boolean inMethod, SymbolResolver sym)
       {
          if (ast == null)
          {
             // nothing to do
             return;
          }

          try
          {
             boolean isEnum = ast.getType() == ENUM_VALUE;
             if (clsDef != null      &&
                 clsDef.isBuiltIn()  &&
                 def != null         &&
                 def.getId() != null &&
                 def.getType() != ENUM_VALUE)
             {
                ast.putAnnotation("refid", def.getId());
             }
             else if (tempIdx != -1)
             {
                ast.putAnnotation("tempidx", Long.valueOf(tempIdx));
                if (clsDef != null)
                {
                   ast.putAnnotation("tempidx-file", clsDef.getFilename());
                }
             }

             if (tokenType == VAR_CLASS || tokenType == FIELD_CLASS)
             {
                if (cls == null)
                {
                   String msg = String.format("WARNING: annotateOptions cannot annotate Class Ref (class==null), " +
                                              "ast=\n%s",
                                              ast.dumpTree(true));
                   System.out.println(msg);
                }
                else
                {
                   sym.annotateClassRef(cls, ast);
                }

                if (dotNetArray)
                {
                   ast.putAnnotation("dotnet-array", true);
                }
                else
                {
                   ast.removeAnnotation("dotnet-array");
                }

                if (genericType != null)
                {
                   ast.putAnnotation("generic-type-parameter", genericType);
                   ast.putAnnotation("generic-type-is-primitive", genericPrim);
                }
             }

             if (builtin)
             {
                ast.putAnnotation("builtin", true);
             }

             if (!original && clsDef != null)
             {
                ast.putAnnotation("found-in-cls", clsDef.getName());
                ast.putAnnotation("found-in-source-file", clsDef.getFilename());
             }

             if (isProp)
             {
                ast.putAnnotation("cls-property", true);
                ast.putAnnotation("cls-container", clsDef.getFilename());
             }

             if (original                          &&
                 !inMethod                         &&
                 ast.getType() != DEFINE_PARAMETER &&
                 ast.getType() != PARAMETER        &&
                 clsDef != null)
             {
                // set default access mode (it will be overridden below if there is an
                // explicit mode specified
                access = isProp || isEnum || ast.getType() == DEFINE_EVENT ? KW_PUBLIC : KW_PRIVATE;
                ast.putAnnotation("access-mode", Long.valueOf(access));
             }

             if (!original && ast.getType() == DATA_SET)
             {
                Aast ds = def.getImmediateChild(SYMBOL, null);

                if (ds == null)
                {
                   throw new RuntimeException(String.format("Malformed dataset definition %s.", def.dumpTree()));
                }

                String sig = (String) ds.getAnnotation("db_signature");

                if (sig == null)
                {
                   throw new RuntimeException(String.format("Missing 'db_signature' annotation in dataset " +
                                                            "definition %s.",
                                                            def.dumpTree()));
                }

                ast.putAnnotation("db_signature", sig);
             }

             if (!original && access != -1)
             {
                // access mode matters on the original definition, propagate it here
                ast.putAnnotation("access-mode", Long.valueOf(access));
             }

             if (isStatic || isEnum)
             {
                ast.putAnnotation("static", true);
             }

             if (isOverride)
             {
                ast.putAnnotation("override", true);
             }

             if (isAbstract)
             {
                ast.putAnnotation("abstract", true);
             }

             if (extent != 0)
             {
                ast.putAnnotation("extent", Long.valueOf(extent));
             }

             if (isEnum)
             {
                def = ast;
                ast.putAnnotation("name", name);
                ast.putAnnotation("type", Long.valueOf(tokenType));
                return;
             }

             // the following annotations are only stored in the original defining
             // AST rather than in references
             if (!original || def == null)
             {
                // nothing else to do
                return;
             }

             // storing the name and type in annotations makes variable definitions
             // easier to generate at conversion time
             if (name != null)
             {
                ast.putAnnotation("name", name);
             }

             if (tokenType != -1)
             {
                ast.putAnnotation("type", Long.valueOf(tokenType));
             }

             Aast child        = (Aast) def.getFirstChild();
             Aast grandchild;

             // Keep track of number of initializers, in case we need to create an annotation for
             // extent, which may not have a constant. In such a case, the number of initializers
             // become the size of the fixed extent.
             // We also moved the ext scope to this level, so we can make a proper decision of the
             // extent value outside of the options processing while loop.
             boolean containsExtent = false;
             int initcount = -1;
             int ext = -1;  // an unspecified extent is -1 (dynamic-extent)

             while (child != null)
             {
                int ctype = child.getType();

                switch (ctype)
                {
                   case KW_SHARED:
                      ast.putAnnotation("shared", Boolean.TRUE );

                      boolean glob = def.getImmediateChild(KW_GLOBAL, null) != null;
                      boolean newed  = def.getImmediateChild(KW_NEW, null) != null;

                      ast.putAnnotation("global",
                                        glob ? Boolean.TRUE : Boolean.FALSE);
                      ast.putAnnotation("newed",
                                        newed ? Boolean.TRUE : Boolean.FALSE);

                      break;
                   case KW_INPUT:
                   case KW_IN_OUT:
                   case KW_OUTPUT:
                      ast.putAnnotation("parmtype", Long.valueOf(ctype));
                      break;
                   case KW_PUBLIC:
                   case KW_PROTECTD:
                   case KW_PRIVATE:
                      // remember the explicit type
                      access = ctype;
                      ast.putAnnotation("access-mode", Long.valueOf(ctype));
                      break;
                   case KW_STATIC:
                      ast.putAnnotation("static", true);
                      break;
                   case KW_ABSTRACT:
                      ast.putAnnotation("abstract", true);
                      break;
                   case KW_OVERRIDE:
                      ast.putAnnotation("override", true);
                      break;
                   case KW_FORMAT:
                      grandchild = (Aast) child.getFirstChild();
                      String fmt = grandchild.getFirstChild().getText();

                      int dtype = def.getType();

                      // format strings that come from fields need extra processing
                      if (dtype > BEGIN_FIELDTYPES || dtype < END_FIELDTYPES)
                      {
                         // although format strings with tildes in 4GL source code need to be coded
                         // as two tildes so that one remains; BUT for fields this is NOT the case!
                         // if the format string comes from a permanent database schema the 4GL seems
                         // to implicitly double the tildes, we simulate that here
                         // TODO: should we exclude temp-table fields from this case?
                         fmt = fmt.replaceAll("~", "~~");
                      }

                      ast.putAnnotation("format", fmt);
                      break;
                   case KW_HELP:
                      ast.putAnnotation("help", child.getFirstChild().getText());
                      break;
                   case KW_INIT:
                      String name     = "initial";
                      int  j          = 0;
                      grandchild = (Aast) child.getFirstChild();

                      while (grandchild != null)
                      {
                         int     gtype   = grandchild.getType();
                         String  gtxt    = grandchild.getText();
                         boolean unknown = (gtype == UNKNOWN_VAL);

                         switch (tokenType)
                         {
                            case VAR_DEC:
                               // this will handle NUM_LITERAL and DEC_LITERAL which
                               // are both valid initializers in Progress
                               Double d = null;

                               if (!unknown)
                               {
                                  if (gtype == NUM_LITERAL || gtype == DEC_LITERAL)
                                  {
                                     boolean isInt = (gtype == NUM_LITERAL);

                                     // cleanup the text to make it a valid input in Java
                                     gtxt = NumberType.parseDecimal(gtxt, '.', ',', isInt);

                                     // save our changes
                                     grandchild.setText(gtxt);

                                     d = Double.valueOf(gtxt);
                                  }
                                  else if (gtype == HEX_LITERAL)
                                  {
                                     d = decimal.fromUnsignedHexLiteral(gtxt).doubleValue();
                                  }
                                  else if (gtype == STRING)
                                  {
                                     gtxt = character.progressToJavaString(gtxt);

                                     if (gtxt.length() == 0)
                                     {
                                        d = 0.0d;
                                     }
                                     else
                                     {
                                        d = Double.valueOf(gtxt);
                                     }
                                  }
                                  else if (gtype == BOOL_TRUE || gtype == BOOL_FALSE)
                                  {
                                     // Progress silently accepts this initializer, we
                                     // assume it is the same as 0 (no init), but this
                                     // is untested at this time.
                                     d = 0.0d;
                                  }
                               }

                               ast.putAnnotation(name, d, j);
                               break;
                            case VAR_INT:
                            case VAR_INT64:
                               Long l = null;

                               if (!unknown)
                               {
                                  if (gtype == DEC_LITERAL)
                                  {
                                     // Progress allows decimal initializers for
                                     // integers, in this case we must round to the
                                     // nearest integer...
                                     Double dd = Double.valueOf(gtxt);
                                     l = Long.valueOf(Math.round(dd.doubleValue()));
                                  }
                                  else if (gtype == NUM_LITERAL)
                                  {
                                     l = Long.valueOf(gtxt);
                                  }
                                  else if (gtype == HEX_LITERAL)
                                  {
                                     l = decimal.fromUnsignedHexLiteral(gtxt).longValue();
                                  }
                                  else if (gtype == STRING)
                                  {
                                     gtxt = character.progressToJavaString(gtxt);

                                     if (gtxt.length() == 0)
                                     {
                                        l = 0l;
                                     }
                                     else
                                     {
                                        l = Long.valueOf(gtxt);
                                     }
                                  }
                                  else if (gtype == BOOL_TRUE ||
                                           gtype == BOOL_FALSE)
                                  {
                                     // Progress silently accepts this initializer,
                                     // we assume it is the same as 0 (no init), but
                                     // this is untested at this time.
                                     l = 0l;
                                  }
                               }

                               ast.putAnnotation(name, l, j);
                               break;
                            case VAR_LOGICAL:
                               // fixup stupid Progress "feature"
                               if (gtype == STRING)
                               {
                                  try
                                  {
                                     booleanManhandle(def, grandchild);
                                  }

                                  catch (AstException ae)
                                  {
                                     LOG.log(Level.SEVERE, "", ae);
                                  }
                               }

                               Boolean b = null;

                               if (!unknown)
                               {
                                  b = (grandchild.getType() == BOOL_TRUE ?
                                       Boolean.TRUE : Boolean.FALSE);
                               }

                               ast.putAnnotation(name, b, j);
                               break;
                            case VAR_CHAR:
                            case VAR_DATE:
                               String s = unknown ?
                                          null    :
                                          new String(grandchild.getText());

                               ast.putAnnotation(name, s, j);
                               break;
                         }
                         j++;
                         grandchild = (Aast) grandchild.getNextSibling();
                      }
                      // Grab the number of initializers in case extent needs to become fixed
                      initcount = j;
                      break;
                   case KW_FONT:
                      long font = 0;
                      if (child.downPath(EXPRESSION, NUM_LITERAL))
                      {
                         font = Long.parseLong(child.getFirstChild().getFirstChild().getText());
                      }
                      else if (child.downPath(EXPRESSION, HEX_LITERAL))
                      {
                         String gtxt = child.getFirstChild().getFirstChild().getText();
                         font = decimal.fromUnsignedHexLiteral(gtxt).longValue();
                      }

                      ast.putAnnotation("font", font);
                      break;
                   case KW_CTX_H_ID:
                      long ctxHelpID = 0;
                      if (child.downPath(EXPRESSION, NUM_LITERAL))
                      {
                         ctxHelpID = Long.parseLong(child.getFirstChild().getFirstChild().getText());
                      }
                      else if (child.downPath(EXPRESSION, HEX_LITERAL))
                      {
                         String gtxt = child.getFirstChild().getFirstChild().getText();
                         font = decimal.fromUnsignedHexLiteral(gtxt).longValue();
                      }

                      ast.putAnnotation("context-help-id", ctxHelpID);
                      break;
                   case KW_LABEL:
                      int  i          = 0;
                      grandchild = (Aast) child.getFirstChild();

                      while (grandchild != null)
                      {
                         ast.putAnnotation("label", grandchild.getText(), i);
                         i++;
                         grandchild = (Aast) grandchild.getNextSibling();
                      }
                      break;
                   case KW_COL_LAB:
                      ast.putAnnotation("columnlabel", child.getFirstChild().getText());
                      break;
                   case KW_DECIMALS:
                      int prec = ExpressionConversionWorker.literalToInt(child.getChildAt(0));
                      if (tokenType == VAR_DEC && prec != 10)
                      {
                         ast.putAnnotation("decimals", Long.valueOf(prec));
                      }
                      break;
                   case KW_EXTENT:
                      Aast extChild = child.getChildAt(0);

                      if (extChild != null)
                      {
                         ext = ExpressionConversionWorker.literalToInt(extChild);
                      }
                      // Indicate there was an EXTENT keyword
                      containsExtent = true;
                      break;
                   case KW_NOT:
                      if (child.getFirstChild().getType() == KW_CASE_SEN)
                      {
                         ast.putAnnotation("casesens", Boolean.FALSE);
                      }
                      break;
                   case KW_CASE_SEN:
                      ast.putAnnotation("casesens", Boolean.TRUE);
                      break;
                   case KW_NO_UNDO:
                      ast.putAnnotation("undoable", Boolean.FALSE);
                      break;
                   case KW_SERIALAB:
                      ast.putAnnotation("serializable", Boolean.TRUE);
                      break;
                   case KW_SERIALZH:
                      ast.putAnnotation("serializable", Boolean.FALSE);
                      break;
                   case KW_SERIALZN:
                      ast.putAnnotation("serializename",
                                        StringHelper.stripEnclosing(child.getFirstChild().getText(), quotes));
                      break;
                   case KW_XML_NNAM:
                      ast.putAnnotation("xmlnodename",
                                        StringHelper.stripEnclosing(child.getFirstChild().getText(), quotes));
                      break;
                }

                child = (Aast) child.getNextSibling();
             }

             // Process any extent options here, since we have to hold off until we've checked
             // for initilizers on a non-specified extent.
             if (containsExtent)
             {
                ext = ((ext == -1) && (initcount != -1)) ? initcount : ext;
                if (ext != 0)
                {
                   this.extent = ext;
                   ast.putAnnotation("extent", Long.valueOf(ext));
                }
             }
          }

          finally
          {
             if (helper != null)
             {
                helper.annotate(this, ast, original, inMethod, sym);
             }
          }
       }

       /**
        * This method provides a fixup for a ridiculous Progress feature of allowing arbitrary
        * user-defined initializer strings.  This processing can't be done during the initial
        * parsing because the format string can appear <b>after</b> the initializer!
        * <p>
        * If no format string is available, then the default is made to be'yes/no' and
        * 'true/false'.  Otherwise the specified format is used.  The core conversion logic
        * is implemented in <code>logical.fromString()</code>.
        * <p>
        * This method will rewrite input AST's token type to be the proper boolean literal
        * token type.
        *
        * @param    grandparent
        *           This is the anscestor 2 levels up from the node being
        *           processed (bogusString). It must be passed in because
        *           this processing occurs during parsing, before the parent
        *           linkages have been fixed up in the tree.  For this reason,
        *           it is not possible to use {@link Aast#getAncestor} or other
        *           more natural approaches to obtain the grandparent.
        * @param    bogusString
        *           The AST node which is a <code>STRING</code> but should be a
        *           <code>BOOL_TRUE</code> or <code>BOOL_FALSE</code>.
        *
        * @throws   AstException
        *           When no match is found.
        */
       public void booleanManhandle(Aast grandparent, Aast bogusString)
       throws AstException
       {
          Aast   format = grandparent.getImmediateChild(KW_FORMAT, null);
          String target = bogusString.getText().toLowerCase();
          String fmt    = null;

          if (format != null)
          {
             fmt = format.getFirstChild().getFirstChild().getText().toLowerCase();
             fmt = StringHelper.stripEnclosing(fmt, quotes);
          }

          target = StringHelper.stripEnclosing(target, quotes);

          Boolean res = logical.fromString(target, fmt);

          if (res != null)
          {
             bogusString.setType(res ? BOOL_TRUE : BOOL_FALSE);
          }
          else
          {
             // this should never happen!
             throw new AstException("VAR_LOGICAL of unknown format (text " +
                                     bogusString.getText() +
                                     ", format string = " + fmt + ") ");
          }
       }

       /**
        * Render the instance as text.
        *
        * @return   The details of this instance as text.
        */
       public String toString()
       {
          return String.format("VARIABLE name %s; token type %d; extent %d; classname %s; AST %s",
                               name,
                               tokenType,
                               extent,
                               cls,
                               def == null ? "null" : def.dumpTree());
       }

       /**
        * Method called to notify that the parse of the file has finished.  This allows the instance to reduce the
        * used memory.
        *
        * @param    global
        *           Flag indicating that the parsing of all files has finished, and a global cleanup can be done
        *           for this instance.
        */
       public void parseFinished(boolean global)
       {
          if (def == null)
          {
             return;
          }

          if (global)
          {
             def = null;
          }
          else
          {
             def = def.duplicate();
          }
       }

       /**
        * Get the {@link #def} ASt.
        *
        * @return   See above.  WARNING: do not alter this AST.
        */
       Aast getDefinition()
       {
          return def;
       }

       /**
        * Helper to copy a child subtree of a specific type from the source to
        * the target AST if the target AST does not already have a child of that
        * type.
        *
        * @param    source
        *           The AST to copy from.
        * @param    target
        *           The AST to copy to.
        * @param    type
        *           The token type of the child tree to copy if it exists in the
        *           source tree and does not already exist in the target tree.
        *
        * @return   <code>true</code> if a copy occurred.
        */
       static boolean overrideIfNotPresent(Aast source, Aast target, int type)
       {
          Aast child = target.getImmediateChild(type, null);

          if (child == null)
          {
             child = source.getImmediateChild(type, null);

             if (child != null)
             {
                try
                {
                   child.graftCopyTo(target, (int[]) null, false);
                }

                catch (AstException ae)
                {
                   // unexpected failure
                   return false;
                }

                return true;
             }
          }

          return false;
       }

       /**
        * Provides a mechanism to specify custom annotations logic for a given variable instance.
        */
       @FunctionalInterface
       public static interface AnnotationHelper
       {
          /**
           * Applys custom annotation processing to the given node.
           *
           * @param    var
           *           The variable instance being used as the source of the annotations.
           * @param    ast
           *           The AST to annotate.
           * @param    original
           *           <code>true</code> if this is to annotate the AST of the original definition,
           *           <code>false</code> if the AST is only a reference.
           * @param    inMethod
           *           <code>true</code> if this is being done from within a method definition.
           * @param    sym
           *           The current symbol resolver.
           */
          public void annotate(Variable var, Aast ast, boolean original, boolean inMethod, SymbolResolver sym);
       }
    }