character.java

/*
** Module   : character.java
** Abstract : Progress 4GL compatible character object
**
** Copyright (c) 2004-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ----------------------------------Description-----------------------------------
** 001 GES 20050517   @21295 Created initial version with full support
**                           for all Progress character processing
**                           including the concatination, begins, matches
**                           operators (but not CONTAINS which is not
**                           needed at this time), all builtin functions,
**                           case-sensitivity and the unknown value.
** 002 GES 20050606   @21424 Minor additions for function support.
** 003 GES 20050621   @21440 Added an assign() method to modify this
**                           instance's state based on the result of
**                           an expression of compatible type.
** 004 GES 20050627   @21554 Added replace/overlay methods to provide the
**                           substring/overlay assignment type language
**                           statements.
** 005 GES 20050628   @21565 Fixed typo in method name and added unknown
**                           value processing to toStringMessage().
** 006 SVG 20050720   @21768 Cloning support for BaseDataType added.
** 007 NVS 20050728   @21908 Added implementation of PROGRESS builtin fun-
**                           ction CAN-DO as public static canDo, taking
**                           either one or two parameters. One parameter
**                           version is just a catcher for untranslated
**                           CAN-DO calls, throwing an exception. Two par-
**                           ameter version is a real code which is typic-
**                           ally used as a utility function.
** 008 GES 20050728   @21920 Added support for the _cbit() built-in 
**                           function (see the testBitAt() method). Also
**                           changed the name of canDo to matchesList
**                           which is more generic and less like a
**                           Progress security-oriented function.
** 009 GES 20050815   @22118 Added Undoable interface support.
** 010 GES 20050825   @22253 Fixed toStringExport() to double up double
**                           quotes appearing inside the string.
** 011 GES 20050826   @22270 Implement case-sensitivity in output from
**                           specific functions and operators depending
**                           on the case-sensitivity of character operands
**                           or parameters.
** 012 GES 20050828   @22279 Added support for temporarily setting
**                           case-sensitivity with an automatic reset at
**                           the end of the scope in which the setting
**                           was made.
** 013 GES 20050829   @22284 Split assignment of configuration and state
**                           from normal assign(). deepAssign() is now 
**                           provided to set both configuration and state
**                           at the same time.
** 014 GES 20050829   @22371 Added error condition support.
** 015 GES 20050907   @22594 Added "raw" type support to substring().
** 016 ECF 20051004   @22958 Implement hashCode() method. Required to be
**                           consistent with BaseDataType's implementation
**                           of equals() method.
** 017 GES 20051024   @23122 Added convertToSQLLike() method.
** 018 ECF 20051104   @23237 Added iterate() method to SensitivityResetter
**                           inner class. No-op implementation; required
**                           to satisfy expanded Finalizable interface.
** 019 GES 20060112   @23892 Added additional valueOf signatures for the
**                           literal forms of the string(), fill(),
**                           entry(), indexOf() and lastIndexOf()
**                           functions.
** 020 GES 20060114   @23903 Fixed substring (2nd parm was used wrong).
** 021 GES 20060206   @24297 Made compareTo() safe to process any object
**                           type (avoids ClassCastException).
** 022 GES 20060208   @24387 Expose a format string expansion helper as
**                           public and added a default format string
**                           getter.
** 023 GES 20060303   @24882 Don't escape the single quote char in Java 
**                           strings.
** 024 GES 20060312   @24985 New variants for overlay and replace to
**                           handle all possible combos with primitives.
** 025 ECF 20060322   @25203 Honor case sensitivity in indexOf() methods.
**                           Added new indexOf variant to which all others
**                           delegate.  Performs scan with proper case
**                           sensitivity setting.
** 026 GES 20060403   @25318 New variants for lookup().
** 027 GES 20060410   @25457 New variants for substring().
** 028 GES 20060417   @25552 Added default value generation.
** 029 GES 20060420   @25620 Added new signatures for right/leftTrim.
** 030 GES 20060422   @25650 Changed assignment style syntax backing
**                           methods to return "this" such that the result
**                           can be assigned back in the case of a 
**                           database field. Added entry statement 
**                           support.
** 031 GES 20060427   @25762 New signatures for trim().
** 032 GES 20060427   @25767 New signature for length().
** 033 GES 20060508   @26023 Fix for Progress pattern to regular 
**                           expression conversion routine
**                           (convertToRegEx).
** 034 ECF 20060510   @26062 Introduced a runtime variant of static method
**                           convertToSQLLike(). This variant accepts a
**                           character parameter.
** 035 GES 20060517   @26192 Match new Finalizable interface requirement.
** 036 GES 20060518   @26280 Safety code for negative start indexes in
**                           substring.
** 037 GES 20060525   @26530 Fix for asc() which needed a 0-based index.
** 038 ECF 20060613   @27243 Fix for asc(). Replaced implementation to
**                           convert the low order character byte to an
**                           unsigned integer.
** 039 GES 20060724   @28158 Added a forced assign() abstract method.
** 040 GES 20060807   @28471 Moved to externalizable interface to optimize
**                           network performance.
** 041 ECF 20060812   @28586 Added methods needed for database server-side
**                           function support.
** 042 GES 20060904   @29214 Added helpers to space-ify strings with an
**                           embedded null character or null char escape
**                           sequence.
** 043 GES 20060905   @29222 Added embedded null processing for the
**                           toString() and concat() variants. Fixed
**                           null byte processing for asc().
** 044 GES 20060905   @29228 Added safeDuplicate().
** 045 GES 20060907   @29282 Fixed substring processing in "raw" mode.
** 046 GES 20060908   @29346 Fixed lookup() to return 0 instead of -1 on
**                           failure.
** 047 ECF 20060908   @29384 Made constructors which accept a J2SE String
**                           null-safe. A null string argument results in
**                           the instance being set to unknown value.
** 048 GES 20061109   @31050 Added formatLength().
** 049 ECF 20070306   @32284 Added case-sensitive variants of equals and
**                           hashCode. These are equalsCaseSensitively and
**                           caseSensitiveHashCode. Also modified existing
**                           hashCode method to be more consistent with
**                           equals. The previous implementation was not
**                           consistent with equals for case-insensitive
**                           character instances.
** 050 GES 20070321   @32504 Fix for some forms of lastIndexOf() which
**                           were starting the backwards search from the
**                           beginning of the source string instead of the 
**                           end as it should have been. This affected
**                           all forms of the method which only have 2
**                           parameters. Also fixed replaceAll() which
**                           would fail to replace two back-to-back
**                           instances of the same target string (the
**                           first would be replaced and the first char
**                           of the second match would be dropped).
** 051 GES 20070926   @35243 Provide external control over 4GL compatible
**                           escape characters when processing non-runtime
**                           strings in progressSpacifyNull(). On Windows
**                           that code should not treat the backslash as
**                           an escape character.
** 052 GES 20080312   @37442 Fixes and documentation added to format
**                           string processing. Fill characters and
**                           padding were not always being handled
**                           properly. Data characters were not being
**                           copied into the right locations in the output
**                           string when non-trailing fill characters were
**                           present.
** 053 GES 20080414   @38012 Fixed chr/asc to support all non-DBCS
**                           charsets. Added fromExportString().
** 054 ECF 20080804   @39286 Added 2 new flavors of numEntries(). Needed
**                           to handle combinations of character and
**                           String parameters.
** 055 GES 20080909   @39766 Moved concat() to varargs. This makes it
**                           possible to merge multiple concat operator
**                           usage into a single method call, which is
**                           significantly more efficient and cleaner.
** 056 GES 20080911   @39788 Added assign(String) signature.
** 057 GES 20081002   @40033 Modified method names to shorten calling
**                           code.
** 058 SVL 20081224   @40985 valueOf(..) returns unknown character value
**                           if converted data is an unknown value.
** 059 ECF 20090212   @41296 Made class final.
** 060 GES 20090422   @41905 Converted to standard string formatting.
** 061 GES 20090424   @41968 Import change.
** 062 GES 20090706   @43106 Added signatures for overlay().
** 063 ECF 20090814   @43621 Fixed various methods which use delimiters. Added
**                           replaceEntry(double, character, String) variant.
** 064 ECF 20090816   @43659 Rolled back #059 (@41296).
** 065 GES 20110303          Moved maximum/minimum to varargs.
** 066 GES 20110422          Fixed case-sensitivity inheritance in the return
**                           value of concat().
** 067 SVL 20110622          Modified convertToSQLLike, added
**                           convertToSQLBegins, postprocessStringLiteral,
**                           toUpperCase(String, boolean) and
**                           substring(String, int).
** 068 GES 20110801          Modified substring() signatures to use NumberType.
** 069 CS  20121119          Implement the quoter built-in function.
** 070 CS  20121231          Added String overloads methods for quoter 
**                           built-in function
** 071 EVL 20130107          Changing progressToJavaString() adding the version
**                           of the method that makes special processing for
**                           '\' path related strings for Windows-target case.
** 072 CA  20130123          Added c'tor which builds a new instance from a
**                           BaseDataType value, if possible. This should be
**                           used only when constructing an instance using the
**                           value returned by a DYNAMIC-FUNCTION call; 4GL
**                           does some special casting in this case, which is
**                           not implemented yet.
** 073 OM  20121023          Added support for TO-ROWID(STRING(row_id)).
** 074 AIL 20130202          Moved static methods common to character and 
**                           longchar to TextOps class.
** 075 OM  20130212          Added support CONTAINS clause.
** 076 GES 20130322          Fixed the BDT c'tor javadoc.
** 077 OM  20130422          Added support for int64 data-type.
** 078 EVL 20140310          Change in progressToJavaString() to consider both "\\" and "\\\\"
**                           characters as single file separator for Windows target.
** 079 OM  20140917          Moved SQL string conversions functions to SQLHelper class.
**                           Added support for pipeline processing in small steps:
**                           ProgressParser, HQLConversion/Preprocessing, Java-String conversion.
**                           Updated integrated test-cases with Windows (non unix-escape) set.
** 080 OM  20141024          Fixed case-sensitivity of valueOf(Text), quoterWorker().
** 081 GES 20141003          Moved some helper methods to parent class and added forceBytes()
**                           to allow deliberate bypass of embedded null character protection.
**                           Improved or added explicit null character handling as needed.
** 082 OM  20141118          Fixed quoterWorker() regression.
** 083 GES 20150131          Corrected format string length calculation to account for unescaped
**                           tildes.
** 084 ECF 20150715          Replace StringBuffer with StringBuilder.
** 085 CA  20151222          Added instantiateDefaultExtent.
** 086 EVL 20160224          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
** 087 IAS 20160331          Fixed potential NPE.
** 088 GES 20160504          Added javaRuntimeToProgressSourceString().
** 089 CA  20160627          Reworked undoable support - the undoables register themselves with
**                           all blocks up the stack, until either 1. the tx block which created
**                           it is reached or 2. the last tx block where it was saved is reached.
** 090 HC  20180223          Improved assignment of SCREEN-VALUE value, the value may contain
**                           fill characters according to the widget format and data type.
** 091 CA  20181106          forceBytes(String) must interpret the bytes using a single-byte
**                           charset.
** 092 CA  20181220          Allow POLY for format argument in STRING builtin function.
**                           Added source and target code-page args for CHR and ASC functions.
** 093 OM  20190206          Added implementation of field size needed by record-length.
**                           Adjsuted valueOf(rowid).
** 094 SBI 20190405          Added truncateValue().
** 095 EVL 20190417          Adding codepage support for CHR/ASC functions.
** 096 GES 20190604          Added additional undocumented quote escaping quirk to string literal
**                           processing.
**     EVL 20190924          Adding Unicode chars support to Progress to Java strings conversion.
**     GES 20191107          Fixed incorrect Unicode escapes on windows. Added support for the
**                           full range of UTF-32 characters which can be encoded in 4GL source
**                           strings (as ~UXXXXXX escape sequences).
** 097 CA  20200304          Allow STRING(javaInstance).
** 098 ME  20200810          Switched to improved I18N workers for chr/asc.
** 099 IAS 20201007          Added Type enum
** 100 EVL 20201222          Variable based on unknown rowid var should be made unknown, not known '?'.
**     VVT 20210317          character valueOf() fixed for the case null passed af the format argument.
**     OM  20210321          Exposed members to LOB classes.
**     VVT 20210331          character valueOf(rowid) fixed: uses rowid.toString(), misleading javadoc
**                           removed.  See #5218.
**     CA  20210609          Reworked INPUT/INPUT-OUTPUT parameters to a new approach, where they are explicitly 
**                           initialized at the method's execution, and not at the caller's arguments.
**     IAS 20210922          Fixed processing of the empty repeat format spec.  
**     HC  20211001          Implementation of i18n support.
**     CA  20211004          An empty string formatted with 'fill chars' must return all-space string.
**     IAS 20211223          No NumberFormatException in the preprocessFormatString() method. 
**     AL2 20220328          Added proxy checks for BDT.
** 101 CA  20230215         'duplicate()' method returns the real type instead of BDT.
** 102 CA  20230526          Added 'forceLengthLimit', which is used to disable character limit enforcement,
**                           when creating a character via 'forceCharacterType'.
** 103 SB  20240411          Changed forceBytes to use the Java equivalent for the used SESSION:CPINTERNAL 
**                           charset. Refs #8468. 
** 104 CA  20241030          Added character constant and cache support - these are immutable instances which 
**                           must not 'escape' from FWD runtime, and only used via assignment or as read-only.
**                           Avoid 'Text.javaSpacifyNull' if the string is known to already have been 
**                           processed.
** 105 ICP 20250129          Used int64.of to leverage caches instances.
**     ICP 20250207          Modified the character constant cache to use CacheManager and have it's
**                           maximum size configurable.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util;

import java.nio.charset.*;
import java.util.*;

import com.goldencode.p2j.persist.*;
import com.goldencode.util.*;

/**
 * A class that represents a Progress 4GL compatible character object whose
 * data is mutable.  All Progress language features related to character
 * types are supported including all functionality that can be accessed via
 * the character concatination operator and builtin Progress string
 * functions.  Normal character comparison operators are implemented in
 * the {@link CompareOps} class. Operator and functions common to character 
 * and longchar types are implemented as statics in the {@link TextOps} 
 * class.
 * <p>
 * In addition to the implementation of the base string processing functions
 * and operators, this class hides the representation of the 
 * <code>unknown value</code> and properly maintains the processing of
 * <code>unknown value</code>. 
 * <p>
 * Format string related processing is handled in this class, see method
 * {@link #toString(String)}.
 * <p>
 * The following is the mapping of Progress language features to the
 * corresponding feature in this class:
 * <p>
 * <pre>
 * contains         operator                       not supported at this time
 * _cbit            function                       {@link #testBitAt}
 * chr              function                       {@link #chr}
 * asc              function                       {@link #asc}
 * string           function                       {@link #valueOf}
 * fill             function                       {@link #fill}
 * quoter           function                       {@link #quoter} 
 * </pre> 
 * <p>
 * <b>Please note that at this time, no conversion of Progress strings to
 * Java strings is done.  All strings passed are expected to be Java
 * strings.  This behavior may need changing in the case where a Progress
 * string is built at runtime but needs to be used as a Java string.</b> In
 * this case the {@link #progressToJavaString} method should be used.
 *
 * @author    GES
 */
public class character
extends Text
{
   /** Default maximum number of cached constant character instances */
   private static final int DEFAULT_CHARACTER_CACHE_LIMIT = 2048;
   
   /** Empty character constant. */
   public static final character EMPTY_STRING = new characterConstant("");
   
   /** Unknown character constant. */
   public static final character UNKNOWN = new characterConstant();

   /** Maximum number of cached integer instances */
   private static int CHARACTER_CACHE_LIMIT;

   /** JVM-wide cache of constant character instances. */
   private static Map<String, character> CHARACTER_CONSTANTS;

   /** The default format string. */
   static final String defaultFormat = "x(8)";

   /**
    * Default constructor, creates an instance that represents the unknown
    * value.
    */
   public character()
   {
      super();
   }

   /**
    * This is a special c'tor which should be used only when converting the
    * value returned by a function or method with polymorphic return type into the
    * expected type (i.e. DYNAMIC-FUNCTION()).  In such cases, the 4GL does some
    * automatic type conversion (this is delegated to the superclass' constructor).
    * 
    * @param    value
    *           The value to be used for this instance.
    */
   public character(BaseDataType value)
   {
      super(value);
   }
   
   /**
    * Create a new instance, by copying the state of the passed {@link Text}.
    * 
    * @param    value
    *           The value which needs to be copied.
    */
   public character(Text value)
   {
      super(value);
   }
   
   /**
    * Constructs an instance that has the <code>String</code> value passed
    * in and which <b>does not</b> represent the <code>unknown value</code>,
    * unless the <code>value</code> argument is <code>null</code>.
    *
    * @param    value
    *           The character value of the resulting instance.
    */
   public character(String value)
   {
      super(value);
   }
   
   /**
    * Constructs an instance that has the <code>String</code> value passed
    * in and which <b>does not</b> represent the <code>unknown value</code>
    * unless the <code>value</code> argument is <code>null</code>.
    *
    * @param    value
    *           The character value of the resulting instance.
    * @param    caseSens
    *           Specifies whether this instance should use case sensitive
    *           comparisons.
    */
   public character(String value, boolean caseSens)
   {
      super(value, caseSens, true);
   }
   
   /**
    * Constructs an instance that has the <code>String</code> value passed
    * in and which <b>does not</b> represent the <code>unknown value</code>
    * unless the <code>value</code> argument is <code>null</code>.
    *
    * @param    value
    *           The character value of the resulting instance.
    * @param    caseSens
    *           Specifies whether this instance should use case sensitive
    *           comparisons.
    * @param    javaSpacifyNull
    *           Flag indicating if {@link #javaSpacifyNull(String)} needs to be executed on the value or not.
    */
   public character(String value, boolean caseSens, boolean javaSpacifyNull)
   {
      super(value, caseSens, javaSpacifyNull);
   }
   
   /**
    * Constructs an instance that has a <code>char</code> value passed
    * in and which <b>does not</b> represent the <code>unknown value</code>. 
    *
    * @param    ch
    *           The character value of the resulting instance.
    */
   public character(char ch)
   {
      super(ch);
   }
   
   /**
    * Constructs an instance that has the <code>String</code> value and
    * <code>unknown value</code> that exactly matches that of the passed-in 
    * instance.
    *
    * @param    value
    *           The character and unknown value of the resulting instance.
    */
   public character(character value)
   {
      super(value);
   }

   /**
    * Initializes the cache using CacheManager. The default value of the cache
    * is used when there is no size available from the configuration.
    */
   public static void initializeCache()
   {
      CHARACTER_CONSTANTS = CacheManager.createIdentityHashMapCache(character.class,
                                                                    "character-cache",
                                                                    DEFAULT_CHARACTER_CACHE_LIMIT);

      CHARACTER_CACHE_LIMIT = CacheManager.getCacheSize(character.class,
                                                        "character-cache",
                                                        DEFAULT_CHARACTER_CACHE_LIMIT);
   }

   /**
    * Retrieve the {@link #CHARACTER_CONSTANTS constant} for the given value.  If it does not exist, create it.
    * <p>
    * The keys in the constant map will be interned - use with care.
    * <p>
    * The result needs to be assigned to a real character instance, and not cached in other maps.
    * 
    * @param    value
    *           The value to retrieve the character constant.
    *           
    * @return   See above.
    */
   public static character of(String value)
   {
      if (value == null)
      {
         return UNKNOWN;
      }
      else if (value.isEmpty())
      {
         return EMPTY_STRING;
      }

      value = value.intern();

      character res = null;
      
      if (CHARACTER_CONSTANTS == null)
      {
         return new characterConstant(value);
      }

      synchronized (CHARACTER_CONSTANTS)
      {
         res = CHARACTER_CONSTANTS.get(value);
      }

      if (res == null)
      {
         res = new characterConstant(value);

         synchronized (CHARACTER_CONSTANTS)
         {
            if (CHARACTER_CONSTANTS.size() < CHARACTER_CACHE_LIMIT)
            {
               CHARACTER_CONSTANTS.put(value, res);
            }
         }
      }

      return res;
   }
   
   /**
    * Returns the numeric value of a character expression which must be
    * only 1 character in length.  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    c
    *           The character expression to convert (must be 1 character in
    *           length).
    * @param    targetCP
    *           The target code page.
    * @param    sourceCP
    *           The source code page.
    *
    * @return    The numeric value of that character which should match up
    *            with the Progress equivalents in the ASCII range.
    */
   public static int64 asc(character c, String targetCP, String sourceCP)
   {
      if (c.isUnknown())
      {
         return int64.UNKNOWN;
      }
      
      return int64.of((long)I18nOps.asc(c.getValue(), targetCP, sourceCP));
   }
      
   /**
    * Returns the numeric value of a character expression which must be
    * only 1 character in length.  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    c
    *           The character expression to convert (must be 1 character in
    *           length).
    *
    * @return    The numeric value of that character which should match up
    *            with the Progress equivalents in the ASCII range.
    */
   public static int64 asc(character c)
   {
      return asc(c, (String)null, (String)null);
   }
      
   /**
    * Returns the numeric value of a string which must be only 1 character
    * in length.  This method will probably generate incorrect results in a
    * DBCS environment since it will use UNICODE as the basis for conversion.
    * If this returned value is different than would be returned under
    * Progress AND if that Progress-specific value is used in comparison with
    * this returned value, then a difference in logic will be seen.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    str
    *           The string to convert (must be 1 character in length).
    *
    * @return   The numeric value of that character which should match up
    *           with the Progress equivalents in the ASCII range.
    */
   public static int64 asc(String str)
   {
      // null embedded characters will be specified by the constructor
      return asc(new character(str), (String) null, (String) null);
   }
   
   /**
    * Returns the numeric value of a string which must be only 1 character
    * in length.  This method will probably generate incorrect results in a
    * DBCS environment since it will use UNICODE as the basis for conversion.
    * If this returned value is different than would be returned under
    * Progress AND if that Progress-specific value is used in comparison with
    * this returned value, then a difference in logic will be seen.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    str
    *           The string to convert (must be 1 character in length).
    * @param    tarCodePage
    *           The target code page.
    *
    * @return   The numeric value of that character which should match up
    *           with the Progress equivalents in the ASCII range.
    */
   public static int64 asc(character str, character tarCodePage)
   {
      String targetCP = (tarCodePage == null || tarCodePage.isUnknown()) ? null : tarCodePage.getValue();
      
      return asc(str, targetCP, null);
   }
   
   /**
    * Returns the numeric value of a string which must be only 1 character
    * in length.  This method will probably generate incorrect results in a
    * DBCS environment since it will use UNICODE as the basis for conversion.
    * If this returned value is different than would be returned under
    * Progress AND if that Progress-specific value is used in comparison with
    * this returned value, then a difference in logic will be seen.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    str
    *           The string to convert (must be 1 character in length).
    * @param    tarCodePage
    *           The target code page.
    * @param    srcCodePage
    *           The source code page.
    *
    * @return   The numeric value of that character which should match up
    *           with the Progress equivalents in the ASCII range.
    */
   public static int64 asc(character str, character tarCodePage, character srcCodePage)
   {
      String targetCP = tarCodePage == null || tarCodePage.isUnknown()
                           ? (String)null : tarCodePage.getValue();
      String sourceCP = srcCodePage == null || srcCodePage.isUnknown()
                           ? (String)null : srcCodePage.getValue();
      
      return asc(str, targetCP, sourceCP);
   }
   
   /**
    * Returns the character value of a numeric expression which must be
    * between 1 and 65534 (inclusive).  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.  The result should match up with the Progress
    * equivalents in the ASCII range.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    i
    *           The int64 to convert.
    *           
    * @return   The associated character or a character representing the
    *           empty string (0 length) when the input is out of range or
    *           <code>unknown value</code> if the input is
    *           <code>unknown value</code>.
    */
   public static character chr(int64 i)
   {
      return chr(i, (String)null, (String)null);
   }
   
   /**
    * Returns the character value of a numeric expression which must be
    * between 1 and 65534 (inclusive).  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.  The result should match up with the Progress
    * equivalents in the ASCII range.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    i
    *           The long to convert.
    *
    * @return   The associated character or a character representing the
    *           empty string (0 length) when the input is out of range.
    */
   public static character chr(long i)
   {
      return chr(i, (String)null, (String)null);
   }
   
   /**
    * Returns the character value of a numeric expression which must be
    * between 1 and 65534 (inclusive).  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.  The result should match up with the Progress
    * equivalents in the ASCII range.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    i
    *           The int64 to convert.
    * @param    tarCodePage
    *           The target code page.
    *           
    * @return   The associated character or a character representing the
    *           empty string (0 length) when the input is out of range or
    *           <code>unknown value</code> if the input is
    *           <code>unknown value</code>.
    */
   public static character chr(int64 i, character tarCodePage)
   {
      String targetCP = tarCodePage == null || tarCodePage.isUnknown()
                           ? (String)null : tarCodePage.getValue();
      
      return chr(i, targetCP, (String)null);
   }
   
   /**
    * Returns the character value of a numeric expression which must be
    * between 1 and 65534 (inclusive).  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.  The result should match up with the Progress
    * equivalents in the ASCII range.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    i
    *           The int64 to convert.
    * @param    tarCodePage
    *           The target code page.
    * @param    srcCodePage
    *           The source code page.
    *           
    * @return   The associated character or a character representing the
    *           empty string (0 length) when the input is out of range or
    *           <code>unknown value</code> if the input is
    *           <code>unknown value</code>.
    */
   public static character chr(int64 i, character tarCodePage, character srcCodePage)
   {
      String targetCP = tarCodePage == null || tarCodePage.isUnknown()
                           ? (String)null : tarCodePage.getValue();
      String sourceCP = srcCodePage == null || srcCodePage.isUnknown()
                           ? (String)null : srcCodePage.getValue();
      
      return chr(i, targetCP, sourceCP);
   }
   
   /**
    * Returns the character value of a numeric expression which must be
    * between 1 and 65534 (inclusive).  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.  The result should match up with the Progress
    * equivalents in the ASCII range.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    i
    *           The int64 to convert.
    * @param    targetCP
    *           The target code page.
    * @param    sourceCP
    *           The source code page.
    *
    * @return   The associated character or a character representing the
    *           empty string (0 length) when the input is out of range or
    *           <code>unknown value</code> if the input is
    *           <code>unknown value</code>.
    */
   public static character chr(int64 i, String targetCP, String sourceCP)
   {
      if (i.isUnknown())
      {
         return new character();
      }
      
      return chr(i.longValue(), targetCP, sourceCP);
   }
      
   /**
    * Returns the character value of a numeric expression which must be
    * between 1 and 65534 (inclusive).  This method will probably generate
    * incorrect results in a DBCS environment since it will use UNICODE
    * as the basis for conversion.  If this returned value is different than
    * would be returned under Progress AND if that Progress-specific value 
    * is used in comparison with this returned value, then a difference in
    * logic will be seen.  The result should match up with the Progress
    * equivalents in the ASCII range.
    * <p>
    * This method does not handle the case of user-specified source and/or
    * target codepages as is possible in Progress.
    *
    * @param    i
    *           The long to convert.
    * @param    targetCP
    *           The target code page.
    * @param    sourceCP
    *           The source code page.
    *
    * @return   The associated character or a character representing the
    *           empty string (0 length) when the input is out of range.
    */
   public static character chr(long i, String targetCP, String sourceCP)
   {
      String ch = I18nOps.chr((int)i, targetCP, sourceCP);
      
      return ch == null ? new character("") : new character(ch);
   }
   
   /**
    * Get the type
    * @return type
    */
   public Type getType()
   {
      return Type.CHARACTER;
   }

   /**
    * Return a character instance that is comprised of <code>num</code>
    * concatenated copies of a given string. 
    *
    * @param    c
    *           The string to be repeated.
    * @param    num
    *           The number of times to repeat.
    *
    * @return   The resulting string or <code>unknown value</code> if any
    *           input is <code>unknown value</code> or an empty string if 
    *           <code>num</code> &lt; 1.
    */
   public static character fill(character c, NumberType num)
   {
      if (c == null || c.isUnknown() || num == null || num.isUnknown())
      {
         return new character();
      }
      
      return fillWorker(c, num.intValue(), c.isCaseSensitive());
   }
   
   /**
    * Return a character instance that is comprised of <code>num</code>
    * concatenated copies of a given string. 
    *
    * @param    c
    *           The string to be repeated.
    * @param    num
    *           The number of times to repeat.
    *
    * @return   The resulting string or <code>unknown value</code> if any
    *           input is <code>unknown value</code> or an empty string if 
    *           <code>num</code> &lt; 1.
    */
   public static character fill(character c, int num)
   {
      if (c == null || c.isUnknown())
      {
         return new character();
      }
      
      return fillWorker(c, num, c.isCaseSensitive());
   }
   
   /**
    * Return a character instance that is comprised of <code>num</code>
    * concatenated copies of a given string. 
    *
    * @param    c
    *           The string to be repeated.
    * @param    num
    *           The number of times to repeat.
    *
    * @return   The resulting string or <code>unknown value</code> if any
    *           input is <code>unknown value</code> or an empty string if 
    *           <code>num</code> &lt; 1.
    */
   public static character fill(character c, double num)
   {
      if (c == null || c.isUnknown())
      {
         return new character();
      }
      
      return fillWorker(c, (int) num, c.isCaseSensitive());
   }
   
   /**
    * Return a character instance that is comprised of <code>num</code>
    * concatenated copies of a given string. 
    *
    * @param    str
    *           The string to be repeated.
    * @param    num
    *           The number of times to repeat.
    *
    * @return   The resulting string or <code>unknown value</code> if any
    *           input is <code>unknown value</code> or an empty string if 
    *           <code>num</code> &lt; 1.
    */
   public static character fill(String str, NumberType num)
   {
      if (num == null || num.isUnknown())
      {
         return new character();
      }
      
      // null embedded characters will be spacified by the constructor
      return fillWorker(new character(str), num.intValue(), false);
   }
   
   /**
    * Return a character instance that is comprised of <code>num</code>
    * concatenated copies of a given string. 
    *
    * @param    str
    *           The string to be repeated.
    * @param    num
    *           The number of times to repeat.
    *
    * @return   The resulting string or <code>unknown value</code> if any
    *           input is <code>unknown value</code> or an empty string if 
    *           <code>num</code> &lt; 1.
    */
   public static character fill(String str, double num)
   {
      // null embedded characters will be spacified by the constructor
      return fillWorker(new character(str), (int) num, false);
   }
   
   /**
    * Return a character instance that is comprised of <code>num</code>
    * concatenated copies of a given string. 
    *
    * @param    str
    *           The string to be repeated.
    * @param    num
    *           The number of times to repeat.
    *
    * @return   The resulting string or <code>unknown value</code> if any
    *           input is <code>unknown value</code> or an empty string if 
    *           <code>num</code> &lt; 1.
    */
   public static character fill(String str, int num)
   {
      // null embedded characters will be spacified by the constructor
      return fillWorker(new character(str), num, false);
   }
   
   /**
    * General quoter function implementation without additional parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code>  or replace it 
    * with '?' if the value is <code>unknown</code>.
    * 
    * @param     var
    *            The <code>BaseDataType</code> variable to operate upon.
    *
    * @return    The resulting quoted character value of the function.
    */
   public static character quoter(BaseDataType var)
   {
      return quoterWorker(var, null, null);
   }
   
   /**
    * General quoter function implementation without additional parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>null</code>  or replace it 
    * with '?' if the value is <code>null</code>.
    * 
    * @param     var
    *            The <code>String</code> variable to operate upon.
    *
    * @return    The resulting quoted character value of the function.
    */
   public static character quoter(String var)
   {
      return quoterWorker(new character(var), null, null);
   }
   
   /**
    * No parameter quoter function implementation.
    * This will be called when trying to quote an unknown literal.
    * 
    * @return    The default unknown character ?.
    */
   public static character quoter()
   {
      return new character("?");
   }
   
   /**
    * Quoter function call for 2 parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code>  or replace it 
    * with '?' if the value is <code>unknown</code>.
    * <p>
    * Additionally a parameter representing a custom  quote character 
    * replacement is provided. If not <code>null</code>, then the first
    * character of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can only be given in the 3 parameter call.
    *  
    * @param    var 
    *           The <code>BaseDataType</code> variable to operate upon
    * @param    customQuote
    *           The <code>String</code> variable
    *           that represents the custom quoter.Custom <code>unknown</code> 
    *           can only be given in 3 parameter call.
    *
    * @return   The resulting quoted character value of the function.
    */
   public static character quoter(BaseDataType var, String customQuote)
   {
      return quoterWorker(var, customQuote, null);
   }
   
   /**
    * Quoter function call for 2 parameters.
    * <p>
    * This puts the <code>String</code> value of the variable between
    * double quotes if the value is not <code>null</code>  or replace it 
    * with '?' if the value is <code>null</code>.
    * <p>
    * Additionally a parameter representing a custom  quote character 
    * replacement is provided. If not <code>null</code>, then the first
    * character of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can only be given in the 3 parameter call.
    *  
    * @param    var 
    *           The <code>String</code> variable to operate upon
    * @param    customQuote
    *           The <code>String</code> variable
    *           that represents the custom quoter.Custom <code>unknown</code> 
    *           can only be given in 3 parameter call.
    *
    * @return   The resulting quoted character value of the function.
    */
   public static character quoter(String var, String customQuote)
   {
      return quoterWorker(new character(var), customQuote, null);
   }
   
   /**
    * Quoter function call for 2 parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code>  or replace it 
    * with '?' if the value is <code>unknown</code>.
    * <p>
    * Additionally a parameter representing a custom character 
    * replacement is provided. If not <code>null</code>, then the first
    * character of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can only be given in the 3 parameter call.
    *  
    * @param    var 
    *           The <code>BaseDataType</code> variable to operate upon
    * @param    customQuote
    *           The <code>String</code> variable
    *           that represents the custom quoter.Custom <code>unknown</code> 
    *           can only be given in 3 parameter call.
    *
    * @return   The resulting quoted character value of the function.
    */
   public static character quoter(BaseDataType var, character customQuote)
   {
      return quoter(var, customQuote, (String) null);
   }
   
   /**
    * Quoter function call for 2 parameters.
    * <p>
    * This puts the <code>String</code> value of the variable between
    * double quotes if the value is not <code>null</code>  or replace it 
    * with '?' if the value is <code>null</code>.
    * <p>
    * Additionally a parameter representing a custom character 
    * replacement is provided. If not <code>null</code>, then the first
    * character of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can only be given in the 3 parameter call.
    *  
    * @param    var 
    *           The <code>String</code> variable to operate upon
    * @param    customQuote
    *           The <code>String</code> variable
    *           that represents the custom quoter.Custom <code>unknown</code> 
    *           can only be given in 3 parameter call.
    *
    * @return   The resulting quoted character value of the function.
    */
   public static character quoter(String var, character customQuote)
   {
      return quoter(new character(var), customQuote, (String) null);
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code> or replace it 
    * with '?' if the value is <code>unknown</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    *  
    * 
    * @param    var
    *           The <code>BaseDataType</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>character</code> quoter variable.
    * @param    customNull
    *           The custom <code>character</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(BaseDataType var,
                                  character customQuote,
                                  character customNull)
   {
      return quoterWorker(var,
                          customQuote.isUnknown() ? null : customQuote.getValue(),
                          customNull.isUnknown()  ? null : customNull.getValue());
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>String</code> value of the variable between
    * double quotes if the value is not <code>null</code> or replace it 
    * with '?' if the value is <code>null</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    *  
    * 
    * @param    var
    *           The <code>String</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>character</code> quoter variable.
    * @param    customNull
    *           The custom <code>character</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(String var,
                                  character customQuote,
                                  character customNull)
   {
      return quoterWorker(new character(var), 
                          customQuote.isUnknown() ? null : customQuote.getValue(),
                          customNull.isUnknown()  ? null : customNull.getValue());
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code> or replace it 
    * with '?' if the value is <code>unknown</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    * 
    * @param    var
    *           The <code>BaseDataType</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>String</code> quoter variable.
    * @param    customNull
    *           The custom <code>character</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(BaseDataType var,
                                  String customQuote,
                                  character customNull)
   {
      return quoterWorker(var, 
                          customQuote, 
                          customNull.isUnknown() ? null : customNull.getValue());
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>String</code> value of the variable between
    * double quotes if the value is not <code>null</code> or replace it 
    * with '?' if the value is <code>null</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    * 
    * @param    var
    *           The <code>String</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>String</code> quoter variable.
    * @param    customNull
    *           The custom <code>character</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(String var,
                                  String customQuote,
                                  character customNull)
   {
      return quoterWorker(new character(var), 
                          customQuote, 
                          customNull.isUnknown() ? null: customNull.getValue());
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code> or replace it 
    * with '?' if the value is <code>unknown</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot;
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    *  
    * @param    var
    *           The <code>BaseDataType</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>character</code> quoter variable.
    * @param    customNull
    *           The custom <code>String</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(BaseDataType var,
                                  character customQuote,
                                  String customNull)
   {
      return quoterWorker(var,
                          customQuote.isUnknown() ? null : customQuote.getValue(),
                          customNull);
   }
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>String</code> value of the variable between
    * double quotes if the value is not <code>null</code> or replace it 
    * with '?' if the value is <code>null</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot;
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    *  
    * @param    var
    *           The <code>String</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>character</code> quoter variable.
    * @param    customNull
    *           The custom <code>String</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(String var,
                                  character customQuote,
                                  String customNull)
   {
      return quoterWorker(new character(var),
                          customQuote.isUnknown() ? null : customQuote.getValue(),
                          customNull);
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>character</code> value of the variable between
    * double quotes if the value is not <code>unknown</code> or replace it 
    * with '?' if the value is <code>unknown</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    *  
    * @param    var
    *           The <code>BaseDataType</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>String</code> quoter variable.
    * @param    customNull
    *           The custom <code>String</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(BaseDataType var, String customQuote, String customNull)
   {
      return quoterWorker(var, customQuote, customNull);
   }
   
   /**
    * Quoter function implementation for 3 parameters.
    * <p>
    * This puts the <code>String</code> value of the variable between
    * double quotes if the value is not <code>null</code> or replace it 
    * with '?' if the value is <code>null</code>.
    * <p>
    * Additionally, a parameter representing a custom quote character
    * is provided.  If not <code>null</code>, then the first character
    * of the given text will be used instead of the &quot; 
    * character.
    * <p>
    * A custom unknown value can also be provided.  If
    * not <code>null</code>, then the given text will be used instead
    * of the default ? <code>unknown</code> replacement to be used.
    *  
    * @param    var
    *           The <code>String</code> variable to operate upon.
    * @param    customQuote
    *           The custom <code>String</code> quoter variable.
    * @param    customNull
    *           The custom <code>String</code> <code>unknown</code> 
    *           replacement to be used.
    *
    * @return   The resulting quoted <code>character</code> value 
    *           of the function.
    */
   public static character quoter(String var, String customQuote, String customNull)
   {
      return quoterWorker(new character(var), customQuote, customNull);
   }
   
   /**
    * Tests a bit in the given position in the input array to determine if
    * that bit is a 0 or 1.  The input string is treated as a multibyte 
    * bitfield (a byte array). The first character of the string corresponds
    * to bit positions 0-7 (if it is non-DBCS and presumably in a DBCS
    * character the positions would be 0-15, although this is not known).  In
    * a non-DBCS charset, the second char would correspond to bits 8-15.
    * <p>
    * This feature supports an undocumented Progress built-in function called
    * _cbit().
    *
    * @param    data
    *           A string with 0 or more characters.
    * @param    index
    *           The bit position to test.
    *
    * @return   The return is always <code>false</code> if the empty string
    *           is input or if a negative index is specified.  For any string
    *           of &gt; 0 bytes length, the return is <code>true</code> if the
    *           bit in the specified position is 1 and <code>false</code> if
    *           it is 0. 
    */
   public static logical testBitAt(character data, int64 index)
   {
      if (data.isUnknown() || index.isUnknown())
      {
         return new logical();
      }
      
      return testBitAt(data, index.intValue());
   }

   /**
    * Tests a bit in the given position in the input array to determine if
    * that bit is a 0 or 1.  The input string is treated as a multibyte 
    * bitfield (a byte array). The first character of the string corresponds
    * to bit positions 0-7 (if it is non-DBCS and presumably in a DBCS
    * character the positions would be 0-15, although this is not known).  In
    * a non-DBCS charset, the second char would correspond to bits 8-15.
    * <p>
    * This feature supports an undocumented Progress built-in function called
    * _cbit().
    *
    * @param    data
    *           A string with 0 or more characters.
    * @param    index
    *           The bit position to test.
    *
    * @return   The return is always <code>false</code> if the empty string
    *           is input or if a negative index is specified.  For any string
    *           of &gt; 0 bytes length, the return is <code>true</code> if the
    *           bit in the specified position is 1 and <code>false</code> if
    *           it is 0. 
    */
   public static logical testBitAt(character data, int index)
   {
      if (data == null || data.isUnknown())
      {
         return new logical();
      }
      
      // invalid/empty data or invalid index
      if (data == null || TextOps.lengthOf(data) == 0 || index < 0)
      {
         return new logical(false);
      }
      
      byte[] bits    = data.getValue().getBytes();
      int    byteOff = index / 8;
      
      // index is too large 
      if (byteOff >= bits.length)
      {
         return new logical(false);
      }
      
      // as with the 4GL, if there are embedded null chars already in the var, this will
      // honor those characters
      byte bitOff = (byte) (0xFF & (0x01 << (index % 8)));
      byte result = (byte) (0xFF & (bits[byteOff] & bitOff));
      
      return new logical(result != 0x00); 
   }

   /**
    * Tests a bit in the given position in the input array to determine if
    * that bit is a 0 or 1.  The input string is treated as a multibyte 
    * bitfield (a byte array). The first character of the string corresponds
    * to bit positions 0-7 (if it is non-DBCS and presumably in a DBCS
    * character the positions would be 0-15, although this is not known).  In
    * a non-DBCS charset, the second char would correspond to bits 8-15.
    * <p>
    * This feature supports an undocumented Progress built-in function called
    * _cbit().
    *
    * @param    data
    *           A string with 0 or more characters.
    * @param    index
    *           The bit position to test.
    *
    * @return   The return is always <code>false</code> if the empty string
    *           is input or if a negative index is specified.  For any string
    *           of &gt; 0 bytes length, the return is <code>true</code> if the
    *           bit in the specified position is 1 and <code>false</code> if
    *           it is 0. 
    */
   public static logical testBitAt(String data, int64 index)
   {
      if (index.isUnknown())
      {
         return new logical();
      }
      
      return testBitAt(data, index.intValue());
   }

   /**
    * Tests a bit in the given position in the input array to determine if
    * that bit is a 0 or 1.  The input string is treated as a multibyte 
    * bitfield (a byte array). The first character of the string corresponds
    * to bit positions 0-7 (if it is non-DBCS and presumably in a DBCS
    * character the positions would be 0-15, although this is not known).  In
    * a non-DBCS charset, the second char would correspond to bits 8-15.
    * <p>
    * This feature supports an undocumented Progress built-in function called
    * _cbit().
    *
    * @param    data
    *           A string with 0 or more characters.
    * @param    index
    *           The bit position to test.
    *
    * @return   The return is always <code>false</code> if the empty string
    *           is input or if a negative index is specified.  For any string
    *           of &gt; 0 bytes length, the return is <code>true</code> if the
    *           bit in the specified position is 1 and <code>false</code> if
    *           it is 0. 
    */
   public static logical testBitAt(String data, int index)
   {
      // protects from null input by using the character constructor
      return testBitAt(new character(data), index);
   }

   /**
    * Convert a <code>double</code> into a character object using
    * the 'message' format for the <code>decimal</code> data type.
    *
    * @param    num
    *           The data to convert.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(double num)
   {
      return valueOf(new decimal(num));
   }
   
   /**
    * Convert an <code>int</code> into a character object using
    * the 'message' format for the <code>int64</code> data type.
    *
    * @param    num
    *           The data to convert.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(long num)
   {
      return valueOf(new int64(num));
   }
   
   /**
    * Convert a <code>String</code> into a character object using
    * the 'message' format for the <code>character</code> data type.
    *
    * @param    s
    *           The data to convert.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(String s)
   {
      return valueOf(new character(s));
   }
   
   /**
    * Convert a <code>boolean</code> into a character object using
    * the 'message' format for the <code>logical</code> data type.
    *
    * @param    b
    *           The data to convert.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(boolean b)
   {
      return valueOf(new logical(b));
   }

   /**
    * Convert a <code>rowid</code> into a character object using
    * the 'message' format for the <code>rowid</code> data type.
    * <p>
    * The returned value is guaranteed to have:
    * <ul>
    *    <li>'0x' prefix
    *    <li>only lowercase hexadecimal digits
    *    <li>even length, but not more that 16 digits
    * </ul>
    *
    * @param    r
    *           The data to convert.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(rowid r)
   {
      if (r.isUnknown())
      {
         return new character();  // not "?", just return unknown
      }
      
      return new character(r.toString());
   }
   
   /**
    * Convert any Progress compatible wrapper into a character object using the 'message' format
    * for the data type being converted.
    *
    * @param   bdt
    *          The data to convert.
    *
    * @return  The resulting string representation.
    */
   public static character valueOf(BaseDataType bdt)
   {
      if (bdt.isUnknown())
      {
         // there is one exception here: string(blob) = "?" not unknown value
         return (bdt instanceof blob) ? new character("?") : new character();
      }
      
      character ret = new character(bdt.toStringMessage(), false, false);
      if (bdt instanceof Text && ((Text) bdt).isCaseSensitive())
      {
         // honor case-sensitivity
         ret.setCaseSensitive(true);
      }
      return ret;
   }
   
   /**
    * Get the string representation of any Java instance.  If this is a {@link BaseDataType},
    * delegate the call to {@link #valueOf(BaseDataType)}.  Otherwise, {@link Object#toString()}.
    * 
    * @param    obj
    *           The instance to get the string representation.
    *           
    * @return   The resulting string representation.
    */
   public static character valueOf(Object obj)
   {
      if (obj instanceof BaseDataType)
      {
         return valueOf((BaseDataType) obj);
      }
      
      return obj == null ? new character() : new character(obj.toString());
   }
   
   /**
    * Convert a <code>double</code> into a character object using
    * the given format for the <code>decimal</code> data type.
    *
    * @param    num
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(double num, String fmt)
   {
      return valueOf(new decimal(num), fmt);
   }
   
   /**
    * Convert an <code>int</code> into a character object using
    * the given format for the <code>int64</code> data type.
    *
    * @param    num
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(long num, String fmt)
   {
      return valueOf(new int64(num), fmt);
   }
   
   /**
    * Convert a <code>String</code> into a character object using
    * the given format for the <code>character</code> data type.
    *
    * @param    s
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(String s, String fmt)
   {
      return valueOf(new character(s), fmt);
   }
   
   /**
    * Convert a <code>boolean</code> into a character object using
    * the given format for the <code>logical</code> data type.
    *
    * @param    b
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(boolean b, String fmt)
   {
      return valueOf(new logical(b), fmt);
   }
   
   /**
    * Convert any Progress compatible wrapper into a character object using
    * the given format for the data type being converted.
    *
    * @param    bdt
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(BaseDataType bdt, String fmt)
   {
      if (bdt.isUnknown() || fmt == null)
         return new character();

      return new character(bdt.toString(fmt));
   }
   
   /**
    * Convert a <code>double</code> into a character object using
    * the given format for the <code>decimal</code> data type.
    *
    * @param    num
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(double num, character fmt)
   {
      return valueOf(new decimal(num), fmt);
   }
   
   /**
    * Convert an <code>int</code> into a character object using
    * the given format for the <code>int64</code> data type.
    *
    * @param    num
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(long num, character fmt)
   {
      return valueOf(new int64(num), fmt);
   }
   
   /**
    * Convert a <code>String</code> into a character object using
    * the given format for the <code>character</code> data type.
    *
    * @param    s
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(String s, character fmt)
   {
      return valueOf(new character(s), fmt);
   }
   
   /**
    * Convert a <code>boolean</code> into a character object using
    * the given format for the <code>logical</code> data type.
    *
    * @param    b
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(boolean b, character fmt)
   {
      return valueOf(new logical(b), fmt);
   }
   
   /**
    * Convert any Progress compatible wrapper into a character object using
    * the given format for the data type being converted.
    *
    * @param    bdt
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(BaseDataType bdt, character fmt)
   {
      return valueOf(bdt, fmt.getValue());
   }
   
   /**
    * Convert a <code>double</code> into a character object using
    * the given format for the <code>decimal</code> data type.
    *
    * @param    num
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(double num, BaseDataType fmt)
   {
      return valueOf(new decimal(num), fmt.toStringMessage());
   }
   
   /**
    * Convert an <code>int</code> into a character object using
    * the given format for the <code>int64</code> data type.
    *
    * @param    num
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(long num, BaseDataType fmt)
   {
      return valueOf(new int64(num), fmt.toStringMessage());
   }
   
   /**
    * Convert a <code>String</code> into a character object using
    * the given format for the <code>character</code> data type.
    *
    * @param    s
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(String s, BaseDataType fmt)
   {
      return valueOf(new character(s), fmt.toStringMessage());
   }
   
   /**
    * Convert a <code>boolean</code> into a character object using
    * the given format for the <code>logical</code> data type.
    *
    * @param    b
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(boolean b, BaseDataType fmt)
   {
      return valueOf(new logical(b), fmt.toStringMessage());
   }
   
   /**
    * Convert any Progress compatible wrapper into a character object using
    * the given format for the data type being converted.
    *
    * @param    bdt
    *           The data to convert.
    * @param    fmt
    *           The format string to use.
    *
    * @return   The resulting string representation.
    */
   public static character valueOf(BaseDataType bdt, BaseDataType fmt)
   {
      if (bdt.isUnknown())
         return new character();

      return new character(bdt.toString(fmt.toStringMessage()));
   }

   /**
    * Removes a set of enclosing double quotes (first and last characters)
    * and converts any doubled up instances (2 instances sequentially) of
    * double quotes into a single instance of double quotes. This reverses
    * the reversible portion of the export format into a normal string as
    * Java would expect to see it.
    *
    * @param    export
    *           A string with the contents in export format as created by
    *           {@link #toStringExport}.
    *
    * @return   A valid Java string.
    */
   public static String fromExportString(String export)
   {
      // this will always work as long as the input string is valid
      String contents = export.substring(1, export.length() - 1);
      
      // remove any doubling up of double quote chars and convert to a single instance
      return contents.replaceAll("\"\"", "\"");
   }
   
   /**
    * Converts a Progress 4GL compatible string to the equivalent Java source 
    * string, by stripping off enclosing quotes/string options and converting
    * the contents (including rewriting all escape sequences). Note that the
    * output format is designed to be written to a Java source file, which
    * requires a different approach than writing a string that can be used 
    * at runtime.
    * <p>
    * Escaped characters (using either the tilde or the backslash character)
    * and special formations will be converted to native representations and
    * any characters that need to be escaped will be so escaped.  In
    * particular, almost all usage of the backslash is compatible between
    * Progress and Java, so only those constructs that must be different
    * will be handled differently.
    * <p>
    * The following conversions will be made:
    * <p>
    * <pre>
    * ~b       to   '\b'   - backspace
    * ~t       to   '\t'   - horizontal tab
    * ~n       to   '\n'   - linefeed
    * ~f       to   '\f'   - formfeed
    * ~r       to   '\r'   - carriage return
    * ~&quot;       to   '\&quot;'   - double quote
    * ~&#x27;       to   '\&#x27;'   - single quote
    * ~\       to   '\\'   - backslash
    * ~~       to   '~'    - single tilde
    * ~{       to   '{'    - left curly brace
    * ~E       to   '\033' - the ASCII escape char
    * ~nnn     to   '\nnn' - octal escape sequence (nnn is a 3 digit octal number)
    * &#x27;&#x27;       to   '\&#x27;'   - single quote (only when inside a
    *                                       string that was enclosed by single
    *                                       quotes)
    * &quot;&quot;       to   '\&quot;'   - double quote (only when inside a
    *                                       string that was enclosed by double
    *                                       quotes)
    * \~       to   '~'    - single tilde 
    * \\       to   '\\'   - backslash
    * \{       to   '{'    - left curly brace
    * \E       to   '\033' - the ASCII escape char
    * </pre>
    * <p>
    * Any other escape sequence that is unrecognized will result in the
    * tilde or backslash being dropped and the following character output
    * unchanged which is the same behavior as Progress.
    * <p>
    * The final result when output will be output inside enclosing double
    * quotes but there is no need for those quotes inside the AST form.
    *
    * @param    progress
    *           The Progress compatible string as parsed from the source
    *           file.
    *
    * @return   A valid Java string that can be written as a string literal.
    */
   public static String progressToJavaString(String progress)
   throws IllegalArgumentException
   {
      return progressToJavaString(progress, false, false);
   }
   
   /**
    * Converts a Progress 4GL compatible string to the equivalent Java source string, by stripping
    * off enclosing quotes/string options and converting the contents (including rewriting all
    * escape sequences). Note that the output format is designed to be written to a Java source
    * file, which requires a different approach than writing a string that can be used at runtime.
    * <p>
    * Escaped characters (using either the tilde or the backslash character) and special
    * formations will be converted to native representations and any characters that need to be
    * escaped will be so escaped.  In particular, almost all usage of the backslash is compatible
    * between Progress and Java, so only those constructs that must be different will be handled
    * differently.
    * <p>
    * The following conversions will be made:
    * <p>
    * <pre>
    * ~b       to   '\b'   - backspace
    * ~t       to   '\t'   - horizontal tab
    * ~n       to   '\n'   - linefeed
    * ~f       to   '\f'   - formfeed
    * ~r       to   '\r'   - carriage return
    * ~&quot;       to   '\&quot;'   - double quote
    * ~'       to   '\''   - single quote
    * ~\       to   '\\'   - backslash
    * ~~       to   '~'    - single tilde
    * ~{       to   '{'    - left curly brace
    * ~E       to   '\033' - the ASCII escape char
    * ~nnn     to   '\nnn' - octal escape sequence (nnn is a 3 digit octal number)
    * ''       to   '\''   - single quote (only when inside a string that was enclosed by single
    *                        quotes)
    * &quot;&quot;       to   '\&quot;'   - double quote (only when inside a string that was 
    *                        enclosed by double quotes)
    * \~       to   '~'    - tilde 
    * \\       to   '\\'   - backslash
    * \{       to   '{'    - left curly brace
    * \E       to   '\033' - the ASCII escape char
    * </pre>
    * <p>
    * Any other escape sequence that is unrecognized will result in the tilde or backslash being
    * dropped and the following character output unchanged which is the same behavior as Progress.
    * <p>
    * The final result when output will be output inside enclosing double quotes but there is no
    * need for those quotes inside the AST form.
    * <p>
    * <strong>Note:</strong><br>
    * Do NOT SQL postprocess the resulting string before it is written to java source!
    *
    * @param   progress
    *          The Progress compatible string as parsed from the source file.
    * @param   windows
    *          The flag indicating the OS is Windows for possible '\' characters.
    * @param   isPathName
    *          The flag indicating the special processing for path elements in Windows.
    *
    * @return   A valid Java string that can be written as a string literal.
    */
   public static String progressToJavaString(String progress, boolean windows, boolean isPathName)
   throws IllegalArgumentException
   {  
      if (progress == null)
      {
         throw new IllegalArgumentException("Invalid input: <null>.");
      }
      
      if (progress.isEmpty())
      {
         throw new IllegalArgumentException("Invalid input: string with no enclosing quotes.");
      }
      
      // are we starting with ' or "
      char lead = progress.charAt(0);
      String contents;
      
      try
      {   
         // this will always work as long as the input string is valid
         contents = progress.substring(1, progress.lastIndexOf(lead));
      }
      catch (IndexOutOfBoundsException iob)
      {
         throw new IllegalArgumentException("Missing closing quote (" + lead + ").");
      }
      
      StringBuilder sb = new StringBuilder();
      int end = contents.length();
      
      // quick exit for the empty string
      if (end == 0)
      {
         return "";
      }
      
      // by now the enclosing quotes are gone, as are any string options
      // so we must convert the contents into the equivalent Java form
      for (int i = 0; i < end; i++) 
      {
         char current = contents.charAt(i);
         char next;
         
         switch (current)
         {
            case '\'':
            case '\"':
               // does the quote type match our enclosing quotes or not?
               if (current == lead)
               {
                  // this should only occur in 2 cases:
                  // 1. "doubled up" quotes and we need to "consume" the extra character
                  // 2. "~" or "\" or '~' or '\' which is the same as " or '
                  
                  char la1 = contents.charAt(i + 1);
                  
                  // the next char is an escape char
                  if (la1 == '~' || (!windows && la1 == '\\'))
                  {
                     // must be case 2
                     int k = i + 2;
                     
                     // confirm the escape char is followed by a matching quote char 
                     if (k < end && contents.charAt(k) == lead)
                     {
                        // ignore current and let the rest process normally
                        continue;
                     }
                     else
                     {
                        throw new IllegalArgumentException("Malformed string (expected a " +
                                                           "matching escaped quote char).");
                     }
                  }
                  
                  i++;
                  next = readChar(contents, i);
                  
                  // must be case 1, check to ensure we aren't smoking something here
                  if (next != current)
                  {
                     throw new IllegalArgumentException("Malformed string (expected doubled " +
                                                        "quote chars).");
                  }
               }
               
               // escape the double quote in java source as in Progress it safely stands alone
               if (current == '\"')
               {
                  sb.append('\\');
               }
               
               sb.append(current);
               break;
            
            case '~':
            case '\\':
               // on Windows we have to make special processing for '\' characters since they are
               // not escape chars
               if (windows && current == '\\')
               {
                  // backslash must be doubled to get one unescaped \ char in a Java source string
                  sb.append("\\\\");
                  
                  // check the next character without increasing the char counter
                  int j = i + 1;
                  
                  // for Windows paths, an extra \ char on input is "eaten"
                  if (isPathName && j < end && readChar(contents, j) == '\\')
                  {
                     i++;
                  }
                  
                  // we must drop out here so that normal escape processing is bypassed
                  break;
               }
               
               // this is an escape char case; there must always be a following character
               i++;
               next = readChar(contents, i);
               
               // handle the special cases that don't have a one-to-one correspondence in Java
               if (next == 'E')
               {
                  // the special case for the ESC char
                  sb.append("\\033");
               }
               else
               {
                  // only certain chars need to be AND in fact other than the
                  // single quote (which we deliberately omit here) only 
                  // these MAY BE escaped in Java so drop the preceding
                  // escape char unless the following char is in this list
                  if (next == 'b'  || next == 't'  || next == 'n'  ||
                      next == 'f'  || next == 'r'  || next == '\"' || 
                      next == '\\' || next == 'u'  || (next >= '0' && next <= '7'))
                  {
                     // we must convert the leading tilde into the Java
                     // escape char because all of these must be escaped in Java
                     sb.append('\\');
                  }
                  
                  // the ~UXXXXXX form has to be processed differently
                  if (next == 'U')
                  {
                     if (!isUnicodeInString(contents, i - 1))
                     {
                        throw new IllegalArgumentException("Malformed supplementary unicode " +
                                                           "escape (" +
                                                           contents.substring(i - 1) +
                                                           ").");
                     }
                     
                     // the direct use of substring here should be safe because of the testing
                     // above (there should be 6 hexadecimal digits to parse in the string)
                     sb.append(convertSupplementaryUnicode(contents.substring(i + 1, i + 7)));
                     i += 6;
                  }
                  else
                  {
                     // insert our following char (warning: this is a bit funky for the octal and
                     // UTF-16 unicode escapes because we don't write the whole thing at once,
                     // but the following chars will all trip the default processing and be
                     // passed through so this works even if it isn't as obvious as we would
                     // like)
                     sb.append(next);
                  }
               }
               break;
            
            default:
               // not an escaped char or special case
               sb.append(current);
         }
      }
      
      return sb.toString();
   }

   /**
    * Converts a Java runtime string into one that can be encoded in Progress 4GL source code
    * as a string literal.  The contents are processed and any values that need escaping are
    * handled and then the result is enclosed in double quote characters.
    * <p>
    * If the runtime Java string is a converted version of a 4GL string literal, then the result
    * of this call is not guaranteed to generate the exact same literal representation. This is
    * due to the fact that Progress has two escape characters (tilde and backslash) on input but
    * on output to Java there is only a single escape character (backslash). For this reason, all
    * mappings are to the tilde escaped version since this will work on UNIX/Linux and Windows
    * in the 4GL.
    * <p>
    * The following conversions will be made:
    * <p>
    * <pre>
    * '\b'           to   ~b        - backspace
    * '\t'           to   ~t        - horizontal tab
    * '\n'           to   ~n        - linefeed
    * '\f'           to   ~f        - formfeed
    * '\r'           to   ~r        - carriage return
    * '\&quot;'      to   ~&quot;   - double quote
    * '\''           to   ~'        - single quote
    * '\\'           to   ~\        - backslash
    * '~'            to   ~~        - tilde
    * '{'            to   ~{        - left curly brace
    * '\033'         to   ~E        - the ASCII escape char
    * control chars  to   ~nnn      - octal escape sequence (nnn is a 3 digit octal number)
    * </pre>
    * <p>
    * The final result when output will be output inside enclosing double quotes.
    *
    * @param    contents
    *           The Java runtime string which should be encoded.
    *
    * @return   A valid Progress string that can be written as a string literal in 4GL source
    *           code.
    */
   public static String javaRuntimeToProgressSourceString(String contents)
   throws IllegalArgumentException
   {  
      if (contents == null)
      {
         throw new IllegalArgumentException("Invalid input: <null>.");
      }
      
      StringBuilder sb = new StringBuilder();
      int end = contents.length();
      
      for (int i = 0; i < end; i++) 
      {
         char current = contents.charAt(i);
         
         switch (current)
         {
            case '\'':
            case '\"':
            case '~':
            case '\\':
            case '{':
               sb.append('~');
               sb.append(current);
               break;
               
            case '\b':
               sb.append("~b");
               break;
               
            case '\f':
               sb.append("~f");
               break;
               
            case '\n':
               sb.append("~n");
               break;
               
            case '\r':
               sb.append("~r");
               break;
               
            case '\t':
               sb.append("~t");
               break;
               
            case '\033':
               // the special case for the ESC char
               sb.append("~E");
               break;
            
            default:
               // check for octal encoding (needed for non-printable stuff, but the 4GL only
               // allows for a single byte to be encoded this way, so we limit it to the set
               // of 3 digit octal values)
               if (Character.isISOControl(current))
               {
                  if ((int)current < 256)
                  {
                     sb.append('~');
                     sb.append(Integer.toOctalString(current));
                  }
                  else
                  {
                     // TODO: what to do here? for now we just skip this char
                  }
               }
               else
               {
                  // not an escaped char or special case
                  sb.append(current);
               }
         }
      }
      
      // we can always encode with containing double quotes because we consistently use tildes
      // to escape all single and double quotes
      return "\"" + sb.toString() + "\"";
   }

   /**
    * Parses a Progress 4GL compatible string, by stripping off enclosing quotes/string options
    * and converting the contents (including rewriting escape sequences).
    * <p>
    * Escaped characters (using either the tilde or the backslash character) and special
    * formations will be converted to native representations and any characters that need to be
    * escaped will be so escaped.
    * <p>
    * The following conversions will be made:
    * <p>
    * <pre>
    * ~b       to   '\b'   - backspace
    * ~t       to   '\t'   - horizontal tab
    * ~n       to   '\n'   - linefeed
    * ~f       to   '\f'   - formfeed
    * ~r       to   '\r'   - carriage return
    * ~&quot;       to   '&quot;'    - double quote
    * ~'       to   '''    - single quote
    * ~\       to   '\'    - backslash
    * ~~       to   '~'    - single tilde
    * ~{       to   '{'    - left curly brace
    * ~E       to   '\033' - the ASCII escape char
    * ~nnn     to   '\nnn' - octal escape sequence (nnn is a 3 digit octal number)
    * ''       to   '''    - single quote (only when inside a string that was enclosed by single
    *                        quotes)
    * &quot;&quot;       to   '&quot;'    - double quote (only when inside a string that was 
    *                        enclosed by double quotes)
    * \~       to   '~'    - tilde 
    * \\       to   '\'    - backslash
    * \{       to   '{'    - left curly brace
    * \E       to   '\033' - the ASCII escape char
    * </pre>
    * <p>
    * Any other escape sequence that is unrecognized will result in the tilde or backslash being
    * dropped and the following character output unchanged which is the same behavior as Progress.
    * <p>
    * The final result must be <code>encodeToJavaSource</code> when it will be output inside
    * enclosing double quotes in a java source file.
    *
    * @param   progress
    *          The Progress compatible string as parsed from the source file.
    *          <code>true</code> if additional escaping for writing the result into a java source
    *          file is needed. If <code>false</code> the result is the normal text representation
    *          of the literal used at runtime or if supplementary processing is needed, in which 
    *          case must be applied.
    *
    * @return  the string value of the input string, unencoded.
    */
   public static String parseProgressCharLiteral(String progress, boolean windows)
   {
      if (progress == null)
      {
         throw new IllegalArgumentException("Invalid input: <null>.");
      }
      
      // remember if we are starting with ' or "
      char lead = progress.charAt(0);
      String contents;
      
      try
      {
         // this will always work as long as the input string is valid
         contents = progress.substring(1, progress.lastIndexOf(lead));
      }
      catch (IndexOutOfBoundsException iob)
      {
         throw new IllegalArgumentException("Missing closing quote (" + lead + ").");
      }
      
      int end = contents.length();
      if (end == 0)// quick exit for the empty string
      {
         return "";
      }
      StringBuilder sb = new StringBuilder();
      
      // the enclosing quotes are gone, so we must parse the contents to remove progress escapes 
      for (int i = 0; i < end; i++)
      {
         char current = contents.charAt(i);
         char next;
         
         switch (current)
         {
            case '\'':
            case '\"':
               // does the quote type match our enclosing quotes or not?
               if (current == lead)
               {
                  // this should only occur if we have "doubled up" quotes and we need to 
                  // "consume" the extra character
                  i++;
                  next = readChar(contents, i);
                  
                  // check to ensure we aren't smoking something here
                  if (next != current)
                  {
                     throw new IllegalArgumentException("Malformed string.");
                  }
               }
               sb.append(current);
               break;
            
            case '\\':
               if (windows)
               {
                  sb.append(current);
                  break;
               }
               // else break trough next case '~', on non-windows \ is also escape character
               
            case '~':
               // after a progress escape, there must always be a following character
               i++;
               next = readChar(contents, i);
               
               // handle the special cases that don't have a one-to-one correspondence in Java
               switch (next)
               {
                  case 'E':
                     // the special case for the ESC char
                     sb.append(033);
                     break;
                  
                  case 'b':
                     sb.append('\b');
                     break;
                  
                  case 't':
                     sb.append('\t');
                     break;
                  
                  case 'n':
                     sb.append('\n');
                     break;
                  
                  case 'f':
                     sb.append('\f');
                     break;
                  
                  case 'r':
                     sb.append('\r');
                     break;
                 
                  case '0':
                  case '1':
                  case '2':
                  case '3':
                  case '4':
                  case '5':
                  case '6':
                  case '7':
                     int octVal = next - '0';
                     if (i + 1 < end)
                     {
                        char secondDigit = contents.charAt(i + 1);
                        if (secondDigit >= '0' && secondDigit <= '7')
                        {
                           i++; // consume the second digit
                           octVal = octVal * 8 + (secondDigit - '0');
                           
                           if (i + 1 < end)
                           {
                              char thirdDigit = contents.charAt(i + 1);
                              if (thirdDigit >= '0' && thirdDigit <= '7')
                              {
                                 i++; // consume the third digit
                                 octVal = octVal * 8 + (thirdDigit - '0');
                                 
                                 // TODO: replace with a 'while' for multibyte char support ?
                              }
                           }
                        }
                     }
                     sb.append((char) octVal);
                     break;
                  
                  default:
                     sb.append(next);
                     break;
               } // end of switch (next)
               break;
            
            default:
               // not an escaped char or special case
               sb.append(current);
         } // switch (current)
      }
      
      return sb.toString();
   }
   
   /**
    * Converts a runtime string to text representation of a java string literal. The following
    * characters are escaped: " and \so they became: \" and respectively \\. The compiler (java
    * lexer) will convert them back when the source file is compiled.
    *
    * The whitespace characters that can be escaped in Java are escaped using \. Characters 
    * outside the ASCII printable are encoded as octal representation.
    * 
    * @param   data
    *          The runtime string.
    *          
    * @return  the string escaped, ready to be written as a java string literal.
    */
   public static String encodeToJavaSource(String data)
   {
      if (data.isEmpty())
      {
         // quick exit for the empty string
         return "";
      }
      
      StringBuilder sb = new StringBuilder();
      int end = data.length();
      
      for (int i = 0; i < end; i++) 
      {
         char current = data.charAt(i);
         switch (current)
         {
            case '"':
               sb.append("\\\"");
               break;
            
            case '\\':
               sb.append("\\\\");
               break;
            
            case '\b':
               sb.append("\\b");
               break;
            
            case '\t':
               sb.append("\\t");
               break;
            
            case '\r':
               sb.append("\\r");
               break;
            
            case '\n':
               sb.append("\\n");
               break;
            
            case '\f':
               sb.append("\\f");
               break;
            
            default:
               if (current > 255)
               {
                  // character is unicode, above ascii interval, output it using hexadecimal
                  String hexStr = Integer.toHexString(current);
                  while (hexStr.length() < 4)
                  {
                     hexStr = "0" + hexStr;
                  }
                  sb.append("\\u").append(hexStr);
               } 
               else if (current < ' ' /*#32*/ || current > '~' /* #127 */)
               {
                  // ascii character is not 'safe', to avoid issues, output it as octal
                  String octStr = Integer.toOctalString(current);
                  while (octStr.length() < 3)
                  {
                     octStr = "0" + octStr;
                  }
                  sb.append("\\").append(octStr);
               }
               else
               {
                  // no processing necessary for the remaining chars
                  sb.append(current);
               }
               break;
         } 
      }
      
      return sb.toString();
   }
   
   /**
    * Convert the Progress 4GL style character type format string into a form that is easier 
    * to process.  All non-escaped 'A(n)' forms are expanded into n instances of A.  All
    * recognized (non-fill) format characters are lowercased in the output to make comparisons
    * easier.
    * <p>
    * DO NOT directly use the result of this expansion for calculation of the format
    * length without also processing any the unescaped tildes that are present in the
    * resulting returned format string.  If you are looking for the format length, please
    * use the <code>formatLength()</code> method of this class.
    *
    * @param    fmt
    *           The Progress 4GL format string for a character data type.
    *
    * @return   The simplified/expanded format string which may still have unescaped tildes.
    */
   public static String preprocessFormatString(String fmt)
   {
      final StringBuilder sb = new StringBuilder();
      
      int  len = fmt.length();
      char c   = (char) 0;
      
      char    last;
      boolean nonFill = false;
      
      for (int i = 0; i < len; i++) 
      {
         last = c;
         c    = fmt.charAt(i);
         
         switch (c)
         {
            case 'x':
            case 'X':
            case 'n':
            case 'N':
            case 'a':
            case 'A':
            case '!':
            case '9':
               if (last != '~')
               {
                  // these are recognized valid 
                  nonFill = true;
                  sb.append(Character.toLowerCase(c));
               }
               else
               {
                  // any fill chars just go into the output unchanged
                  nonFill = false;
                  sb.append(c);
               }
               break;
            case '(':
               if (last != '~' && nonFill)
               {  
                  // find the end of the parens
                  int j = fmt.indexOf(')', i + 1);  
                  
                  if (j < 0)
                  {
                     String spec = "Character format %s is incomplete";
                     String err  = String.format(spec, fmt);
                     
                     ErrorManager.recordOrThrowError(164, err);
                     
                     // this is the safest thing to return (any results
                     // should be undone anyway)
                     return "";
                  }
                  
                  // how many repeats do we have?
                  String nrpt = fmt.substring(i+1, j);
                  int num = 0;
                  try
                  {
                     num = "".equals(nrpt.trim()) ? 0 : Integer.parseInt(nrpt);
                  }
                  catch (NumberFormatException e)
                  {
                     int nc;
                     for(nc = i + 1; nc < j; nc++)
                     {
                        if (!Character.isDigit(fmt.charAt(nc)))
                        {
                           break;
                        }
                     }
                     ErrorManager.recordOrThrowError(22,
                        String.format("Character number %d of format %s is invalid", nc, fmt));
                  }
                  
                  // get our last char
                  char ch = Character.toLowerCase(last);
                  
                  // create and add the resulting string which takes advantage
                  // of the fact that the last char is already in the buffer
                  // and so we only need to repeat (num - 1) times
                  sb.append(StringHelper.repeatChar(ch, num - 1));
                  
                  // move our index up to the end of the section, it will
                  // be incremented past/tested against the string length
                  // by the for loop itself
                  i = j;
               }
               else
               {
                  if (last == '~')
                  {
                     // replace the tilde with this char
                     sb.setCharAt(sb.length() - 1, c);
                  }
                  else
                  {
                     // we must be following a fill char so this is a fill
                     // char too
                     sb.append(c);
                  }
               }
               nonFill = false;
               break;
            default:
               // any fill chars just go into the output unchanged
               nonFill = false;
               sb.append(c);
         }
      }
      
      return sb.toString();
   }

   /**
    * Returns 4GL literal options from the supplied Progress string. The options are returned without the
    * leading colon.
    *
    * @param   progress
    *          Unprocessed progress string.
    *
    * @return  See above.
    */
   public static String getStringOptions(String progress)
   {
      if (progress == null)
      {
         throw new IllegalArgumentException("Invalid input: <null>.");
      }

      // remember if we are starting with ' or "
      char lead = progress.charAt(0);

      // this will always work as long as the input string is valid
      int closingQuote = progress.lastIndexOf(lead);
      if (closingQuote == 0)
      {
         throw new IllegalArgumentException("Missing closing quote (" + lead + ").");
      }

      String opts = progress.substring(closingQuote + 1);
      int colon = opts.indexOf(':');
      if (colon >= 0)
      {
         return opts.substring(colon + 1);
      }
      else
      {
         return null;
      }
   }

   /**
    * Returns string content from the supplied Progress string. I.e. enclosing quotes and any 4GL string
    * literal options with the colon delimiter are not returned in the result.
    *
    * @param   progress
    *          Unprocessed progress string.
    *
    * @return  See above.
    */
   public static String getStringContent(String progress)
   {
      if (progress == null)
      {
         throw new IllegalArgumentException("Invalid input: <null>.");
      }

      // remember if we are starting with ' or "
      char lead = progress.charAt(0);

      // this will always work as long as the input string is valid
      int closingQuote = progress.lastIndexOf(lead);
      if (closingQuote == 0)
      {
         throw new IllegalArgumentException("Missing closing quote (" + lead + ").");
      }

      return progress.substring(1, closingQuote);
   }

   /**
    * Parses the supplied Progress string and returns {@code true} if the string contains the 'U' 4GL
    * string literal option, i.e. the string is untransatable.
    *
    * @param   progress
    *          Unprocessed progress string to test.
    *
    * @return  See above.
    */
   public static boolean isUntranslatable(String progress)
   {
      String opts = getStringOptions(progress);
      return opts != null && (opts.indexOf('u') != -1 || opts.indexOf('U') != -1);
   }
   
   /**
    * This method allows the explicit bypass of embedded null character protections that are
    * in place for all construction and assignment of text values. This is needed because
    * of locations in the 4GL that do allow embedded null characters to leak into variables.
    * <p>
    * The data will be converted to a string and stored as the instance's data. DO NOT USE
    * THIS METHOD except for duplicating the 4GL "null character leaking" behavior.
    *
    * @param    data
    *           The bytes to force in.
    */
   void forceBytes(byte[] data)
   {
      // deliberately bypasses embedded null character protections
      this.value   = new String(data, Charset.forName(I18nOps.getJavaCPInternal()));
      this.unknown = false;
      this.javaSpacifyNull = true;
   }
   
   /**
    * This method allows the explicit bypass of embedded null character protections that are
    * in place for all construction and assignment of text values. This is needed because
    * of locations in the 4GL that do allow embedded null characters to leak into variables.
    * <p>
    * The data will be converted to a string and stored as the instance's data. DO NOT USE
    * THIS METHOD except for duplicating the 4GL "null character leaking" behavior.
    *
    * @param    data
    *           The bytes to force in.
    * @param    length
    *           The number of bytes to copy.
    */
   void forceBytes(byte[] data, int length)
   {
      // deliberately bypasses embedded null character protections
      this.value   = new String(data, 0, length, Charset.forName(I18nOps.getJavaCPInternal()));
      this.unknown = false;
      this.javaSpacifyNull = true;
   }

   /**
    * Returns a clone of the current instance which has all <code>null</code>
    * characters removed from its data (in particular, the data is truncated 
    * starting at any <code>null</code> character).
    *
    * @return   An instance which has no embedded <code>null</code> characters.
    */
   public character safeDuplicate()
   {
      character ch = (character) duplicate();
      
      if (!unknown)
      {
         // this assignment protects from embedded null characters
         ch.value = javaSpacifyNull ? safeValue() : value;
         ch.javaSpacifyNull = false;
      }
      
      return ch;
   }

   /**
    * Returns the instance's value safely truncated at any <code>null</code>
    * byte.
    *
    * @return   A value with no <code>null</code> bytes.
    */
   protected String safeValue()
   {
      return javaTruncateNull(value);
   }
   
   /**
    * Obtain the length (in bytes) of this BDT will use when serialized.
    *
    * @return  the length of this BDT will use when serialized.
    */
   @Override
   public int getSize()
   {
      if (isUnknown())
      {
         return 1;
      }
      int strLen = value.length();
      // the documentation say 240 but, from tests,  the threshold is actually 230
      return 1 + (strLen >= 230 ? (2 + strLen) : strLen);
   }
   
   /**
    * Sets the state (data and unknown value) of this instance based on the state of the passed
    * instance.
    * <p>
    * It will also check the maximum size is not exceeded.
    *
    * @param    value
    *           The instance from which to copy state.
    */
   @Override
   public void assign(Text value)
   {
      // the value may be a proxy; make sure to unwrap and work with the real value
      if (BaseDataType.isProxy(value))
      {
         assign(value.val());
         return;
      }
      
      if (value instanceof longchar && TextOps.byteLengthOf(value) > 31991 && forceLengthLimit()) 
      {
         ErrorManager.recordOrThrowError(9324, "Attempt to exceed maximum size of a CHARACTER variable");
         return;
      }
      
      super.assign(value);
   }

   /**
    * Return the default display format string for this type.
    *
    * @return   The default format string.
    */
   public String defaultFormatString()   
   {
      return defaultFormat;
   }

   /**
    * Calculate the length of any formatted value (the text form of the value)
    * using this format string.
    *
    * @param    fmt
    *           The format string.
    *
    * @return   The length in characters of the any resulting formatted
    *           value.
    */
   public int formatLength(String fmt)
   {
      if (fmt == null)
      {
         fmt = defaultFormatString();
      }
      
      String normalized = preprocessFormatString(fmt);
      
      // this creates a fake result string that fills all valid char positions with a space,
      // drops unescaped tildes and outputs fill characters at that proper positions; the
      // result can be used to detect the actual length of data output using this format 
      String blanked = processDataString("", normalized);
      
      return blanked.length();
   }
   
   /**
    * Does the same as standard <code>clone()</code> method but returns an
    * instance of <code>BaseDataType</code> and doesn't throw the
    * <code>CloneNotSupportedException</code>.
    *
    * @return   A clone of this instance.  
    */
   public character duplicate()
   {
      return new character(this);
   }

   /**
    * Creates a new instance of the same type that represents the 
    * default initialized value.
    *
    * @return   An instance that represents the default value.
    */
   public BaseDataType instantiateDefault()   
   {
      return new character("");
   }

   /**
    * Creates a string representation of the instance data in a form that is
    * compatible with the <code>MESSAGE</code> language statement.  If the
    * instance represents the <code>unknown value</code>, a '?' will be
    * returned.
    *
    * @return   The 'message' formatted string.
    */
   public String toStringMessage()   
   {
      // quick-out when unknown
      if (unknown)
         return "?";
      
      return javaTruncateNull(getValue());
   }

   /**
    * Creates a string representation of the instance data using the 'export'
    * format.  For the character data type the entire string is returned
    * enclosed in a pair of double quotes.  If the instance represents the
    * <code>unknown value</code>, a '?' will be returned.  Any instances of
    * double quotes inside the string will be doubled up on output.
    *
    * @return   The 'export' formatted string.
    */
   public String toStringExport()   
   {
      // quick-out when unknown (no user override for that case)
      if (unknown)
         return "?";
      
      // open the string
      StringBuilder sb = new StringBuilder("\"");
      
      // truncate at any embedded null
      String txt = javaSpacifyNull ? javaTruncateNull(value) : value;
      
      // add the contents, doubling up all instance of double quote chars
      sb.append(txt.replaceAll("\"", "\"\""));
      
      // close the string
      sb.append("\"");
      
      return sb.toString();
   }

   /**
    * Creates a string representation of the instance data with the default
    * Progress 4GL format of 'x(8)'. If the instance represents the
    * <code>unknown value</code>, a '?' will be returned.
    *
    * @return   The formatted string.
    */
   public String toString()   
   {
      return toString(null);
   }

   /**
    * Creates a string representation of the instance data using the user-
    * specified Progress 4GL format string.  If the passed-in format is
    * <code>null</code>, then a default format string of 'x(8)' will be
    * used.  If the instance represents the <code>unknown value</code>,
    * a '?' will be returned.
    *
    * @param    fmt
    *           The Progress 4GL format string for a character data type.
    *
    * @return   The formatted string.
    */
   public String toString(String fmt)   
   {
      // quick-out when unknown (no user override for that case)
      if (unknown)
         return "?";
      
      String clean = null;
      
      // implement 'default' behavior 
      if (fmt == null)
      {
         fmt = defaultFormat;
         
         // default case doesn't spacify
         clean = value;
      }
      else
      {
         // for some reason, when there is an explicit format string, the value is spacified
         clean = javaSpacifyNull ? javaSpacifyNull(value) : value;
      }
      
      // make the format string easier to process
      String format = preprocessFormatString(fmt);

      // trim trailing whitespace off the value
      String trimmed = StringHelper.safeTrimTrailing(clean);
      
      // build the output string and return it
      return processDataString(trimmed, format);
   }

   /**
    * Remove trailing spaces.
    * 
    * @return   This character object with trancated value
    */
   public character truncateValue()
   {
      value = StringHelper.safeTrimTrailingSpaces(value);
      
      return this;
   }

   /**
    * Get the default initialization for an extent variable of this type.
    * 
    * @return   See above.
    */
   @Override
   protected BaseDataType instantiateDefaultExtent()
   {
      return new character("");
   }

   /**
    * Check if length limit must be enforced in {@link #assign(Text)}.
    * 
    * @return   Always true.
    */
   protected boolean forceLengthLimit()
   {
      return true;
   }
   
   /**
    * Convert the input data into an output string that is formatted as
    * specified by the Progress 4GL style character type format string.
    * <p>
    * Recognized format specifier characters are x, X, n, N, a, A, 9 and
    * !. The presence of any unescaped (with tilde) instance of one of these
    * characters in the format string means that should there be a
    * corresponding character (at the same format specifier index) in the
    * data string, it will be copied to the output string at that same
    * position.  No transformation of the data is done except for the !
    * specifier which causes the data character to be uppercased before
    * output.
    * <p>
    * Any non-recognized format specifier and any escaped character
    * (including escaped format specifiers) are considered user-defined
    * "fill" characters.  These characters are copied from the format
    * string (not the data string) into the output string.  They are
    * copied unchanged. If present in the input string they are ignored
    * as non-significant characters. Any unescaped tildes are dropped
    * (not copied to the output).
    * <p>
    * The input data will be copied to a location in the output string based
    * on the simple index position in the input data AND the relative position
    * of each format specifier.  The presence of fill characters does not
    * change the relative position of format specifier characters. This means
    * that the 1st character in the input string will be copied on output
    * to the location of the 1st format specifier character in the format
    * string, which may not be the first position in the output string (if
    * there are fill characters in the format string before the first format
    * specifier). The 2nd character in the input string will be copied on
    * output to the location of the 2nd format specifier, which may not
    * immediately follow the 1st data character in the output string if there
    * are intervening fill characters between the first and second format
    * specifiers.
    * <p>
    * If there is less data characters (in the input string) than format
    * specifier characters (in the format string), then a ' ' (space) will
    * be output for each trailing format specifier position which has no
    * data to be copied.  This is a kind of padding.
    * <p>
    * If there are more data characters (in the input string) than format
    * specifier characters (in the format string), then the output string
    * will be truncated, not copying those extra data characters (in the
    * input string) to the output string.
    * <p>
    * The output string length will always be equivalent to the size of the
    * format string less any dropped tilde escape characters.
    *
    * @param    data
    *           The input string.
    * @param    fmt
    *           The Progress 4GL format string for a character data type.
    *
    * @return   The output (formatted) string.
    */
   static String processDataString(String data, String fmt)
   {
      StringBuilder sb = new StringBuilder();
      
      int  dlen = data.length();
      int  flen = fmt.length();
      char c    = (char) 0;
      
      char last;
      
      // iterate through all characters in the format string (the result
      // will be the same length as the *real* number of characters in the
      // format string --> unescaped tildes don't count) 
      for (int j = 0, i = 0; i < flen; i++) 
      {
         // remember the last character to track if it is a tilde 
         last = c;
         
         // get next format string character
         c = fmt.charAt(i);
         
         switch (c)
         {
            case '~':
               if (last == '~')
               {
                  // we leave a single tilde in the output and reset our
                  // character to ensure that tilde processing is reset
                  sb.append(c);
                  c = (char) 0;
               }
               break;
            case 'x':
            case 'X':
            case 'n':
            case 'N':
            case 'a':
            case 'A':
            case '9':
            case '!':
               // these are recognized valid format characters
               if (last == '~')
               {
                  // this was escaped so it is a fill char, this just goes
                  // into the output unchanged
                  sb.append(c);
               }
               else
               {
                  if (j < dlen)
                  {
                     // there is still data to output
                     
                     // only the ! causes the data to be transformed before output,
                     // all others cause the unchanged char to be output
                     char out = data.charAt(j);

                     if (c == '!')
                        out = Character.toUpperCase(out);

                     sb.append(out);
                     j++;
                  }
                  else
                  {
                     // no more data, append a space to pad the end
                     sb.append(' ');
                  }
               }
               break;
            default:
               // must be an unescaped fill char, add to the output unchanged
               sb.append(dlen == 0 ? ' ' : c);

               // skip the fill char in the input string if present
               if (j < dlen && data.charAt(j) == c)
               {
                  j++;
               }
         }
      }
      
      return sb.toString();
   }

   /**
    * Return a character instance that is comprised of <code>num</code> concatenated copies of a
    * given string.
    *
    * @param   ch
    *          The string to be repeated.
    * @param   num
    *          The number of times to repeat.
    *
    * @return  The resulting string or <code>unknown value</code> if any input is
    *          <code>unknown</code> value or an empty string if <code>num</code> &lt; 1.
    */
   private static character fillWorker(character ch, int num, boolean cs)
   {
      if (num < 1)
      {
         return new character("", cs);
      }
      
      StringBuilder sb  = new StringBuilder();
      String        str = ch.getValue();
      
      // this does copy null chars if they are present, which is the 4GL way
      for (int i = 0; i < num; i++)
      {
         sb.append(str);
      }
      
      return  new character(sb.toString(), cs);
   }
   
   /**
    * This is the core quoter function implementation.  This puts the text value of the variable
    * between double quotes if the value is not unknown, or it will replace it with the 
    * <code>?</code> character if the value is <code>unknown</code>. If the variable is a
    * <code>character</code> type, any contained double quote character will be doubled (another
    * double quote character will be added next to it - Progress behavior).
    * <p>
    * The caller can provide replacement characters to use instead of the double quote character
    * (&quot;) and instead of the default <code>unknown</code> character <code>?</code>.
    * 
    * @param    var
    *           The <code>BaseDataType</code> variable to operate upon.
    * @param    customQuote
    *           The custom quoter character to be used. If <code>null</code>, the default is the
    *           double quote character.
    * @param    customNull
    *           The custom unknown text replacement to be used. If <code>null</code> the default
    *           is '?'.
    *
    * @return   The resulting quoted string or <code>unknown</code>, if the input is
    *           <code>unknown</code>.
    */
   private static character quoterWorker(BaseDataType var, String customQuote, String customNull)
   {
      // custom null character
      String nullVal = (customNull != null) ? customNull : "?";
      
      // custom quoter character (in the 4GL if given a custom character that isn't " or ' it will
      // take the first letter as the custom quote char
      String quoteVal = (customQuote != null) ? customQuote.substring(0, 1) : "\"";
      
      String result = nullVal;
      boolean caseSensitive = false;
      
      if (!var.isUnknown())
      {
         StringBuilder sb = new StringBuilder();
         
         sb.append(quoteVal);
         
         if (var instanceof character)
         {
            String in  = quoteVal;
            String out = quoteVal + quoteVal;
            
            // quote character will be doubled where found inside contents, otherwise
            // it is just the contents truncated at any null chars
            character charVar = (character) var;
            sb.append(javaTruncateNull(charVar.getValue()).replaceAll(in, out));
            caseSensitive = charVar.isCaseSensitive();
         }
         else if (var instanceof date)
         {
            // date format result of quoter is 99/99/9999
            date dateVar = (date) var;
            sb.append(dateVar.toString("99/99/9999"));
         }
         else
         {
            // the rest of the data types format result as the default message format
            sb.append(var.toStringMessage());
         }
         
         sb.append(quoteVal);
         result = sb.toString();
      }
      
      return new character(result, caseSensitive);
   }
   
   /**
    * Converts 6 hexadecimal digits (representing a single UTF-32 supplementary character)
    * into the Java source representation of such characters as two UTF-16 encoded high/low
    * supplementary characters.  If the parsed UTF-32 character can be represented as a
    * single UTF-16 escape sequence (from the basic multilingual plane), then that form
    * will be returned.
    *
    * @param    txt
    *           Text with 6 hexadecimal digits.  DO NOT call this without exactly 6 hexadecimal
    *           digits!
    *
    * @return   The properly encoded unicode in escaped source code form.
    */
   private static String convertSupplementaryUnicode(String txt)
   {
      String encoded = null;
      
      int utf32 = Integer.valueOf(txt, 16);
      
      if (Character.isSupplementaryCodePoint(utf32))
      {
         char high = Character.highSurrogate(utf32);
         char low  = Character.lowSurrogate(utf32);
         
         encoded = String.format("\\u%04X\\u%04X", (int) high, (int) low);
      }
      else
      {
         if (Character.isBmpCodePoint(utf32))
         {
            // short cut: the input must have been in the form 00XXXX so we just use the last 4
            encoded = "\\u" + txt.substring(2);
         }
         else
         {
            throw new IllegalArgumentException("Malformed supplementary char (" + txt + ").");
         }
      }
      
      return encoded;
   }
   
   /**
    * Checks if the given Progress string has unicode character embedded by '~uXXXX', 
    * '~UXXXXXX' (or the two '\' escaped alternatives) formatted string.  Each X is a
    * hexadecimal digit.
    *
    * @param    strToTest
    *           The Progress compatible string to test.
    * @param    startNdx
    *           The character position inside the string to start from.
    *
    * @return   <code>true</code> if Unicode char is matched, <code>false</code> otherwise.
    */
   private static boolean isUnicodeInString(String strToTest, int startNdx)
   {
      // get the string size
      int strLen = strToTest.length();

      // we must check the next 2 chars to know the following size; make sure we have data
      if (strLen - startNdx < 3)
      {
         return false;
      }
      
      // the first char is expected as '\' or '~'
      char nextCh = strToTest.charAt(startNdx);
      
      if (nextCh != '\\' && nextCh != '~')
      {
         return false;
      }
      
      // the second char is expected as 'u' (basic multi-lingual plane) or 'U' (supplementary
      // chars)
      nextCh = strToTest.charAt(startNdx + 1);
      
      if (nextCh != 'u' && nextCh != 'U')
      {
         return false;
      }
      
      int sz = (nextCh == 'u') ? 6 : 8;
      
      // the string size should be 6+ from startNdx
      if (strLen - startNdx < sz)
      {
         return false;
      }
      
      // the next chars are expected to be hexadecimal digits
      for (int i = 2; i < sz; i++)
      {
         nextCh = strToTest.charAt(startNdx + i);
         
         if ((nextCh >= '0' && nextCh <= '9') ||
             (nextCh >= 'A' && nextCh <= 'F') ||
             (nextCh >= 'a' && nextCh <= 'f'))
         {
            continue;
         }
         else
         {
            return false;
         }
      }
      
      return true;
   }
   
   /**
    * Used as comparator for TreeMap using <character>
    */
   public static final Comparator<character> CASE_INSENSITIVE_ORDER = new CaseInsensitiveComparator();

   private static class CaseInsensitiveComparator implements Comparator<character>
   {

      public int compare(character s1, character s2)
      {
        return s1.compareTo(s2);
      }
   }

   /**
    * Encapsulates the logic of storing and executing a set of tests using
    * a minimum set of input (2 string operands and a set of boolean results
    * that are to be expected).
    */
   private static class TestCase
   {
      /** 
       * Use this static flag to switch from unix-escapes to windows mode for backslash during the
       * initialization and then during the tests themselves.
       */
      private static boolean unixEscapes = true;
      
      /** Operand 1. */
      character op1 = null;
      
      /** Operand 2. */
      character op2 = null;
      
      /** Expected test results for begins. */
      boolean begins;
      
      /** Expected test results for equals. */
      boolean eq;
      
      /** Expected test results for notEquals. */
      boolean ne;
      
      /** Expected test results for isGreaterThan. */
      boolean gt;
      
      /** Expected test results for isLessThan. */
      boolean lt;
      
      /** Expected test results for isGreaterThanOrEqual. */
      boolean ge;
      
      /** Expected test results for isLessThanOrEqual. */
      boolean le;
      
      /** Expected test results for matches. */
      boolean matches;
      
      /** Expected test results for case-sensitive begins. */
      boolean cBegins;
      
      /** Expected test results for case-sensitive equals. */
      boolean cEq;
      
      /** Expected test results for case-sensitive notEquals. */
      boolean cNe;
      
      /** Expected test results for case-sensitive isGreaterThan. */
      boolean cGt;
      
      /** Expected test results for case-sensitive isLessThan. */
      boolean cLt;
      
      /** Expected test results for case-sensitive isGreaterThanOrEqual. */
      boolean cGe;
      
      /** Expected test results for case-sensitive isLessThanOrEqual. */
      boolean cLe;
      
      /** Expected test results for case-sensitive matches. */
      boolean cMatches;
      
      /** Save the original Progress string before pre processing in constructor. */
      private String op14gl;

      /** Save the original Progress string before pre processing in constructor. */
      private String op24gl;
      
      TestCase(String op1 , String op2     , boolean begins , boolean eq,
               boolean ne , boolean gt     , boolean lt     , boolean ge,
               boolean le , boolean matches, boolean cBegins, boolean cEq,
               boolean cNe, boolean cGt    , boolean cLt    , boolean cGe,
               boolean cLe, boolean cMatches)
      {
         op14gl = op1;
         op24gl = op2;
         
         // simulate the resulting runtime strings
         String s1 = progressToJavaString("'" + op1 + "'", !unixEscapes, false);
         String s2 = progressToJavaString("'" + op2 + "'", !unixEscapes, false);
         
         this.op1      = new character(StringHelper.processEscapes(s1));
         this.op2      = new character(StringHelper.processEscapes(s2));
         
         
         // check internal implementation
         final String ppl1 = parseProgressCharLiteral("'" + op14gl + "'", !unixEscapes);
         if (!ppl1.equals(this.op1.toStringMessage()))
         {
            System.out.println("Failed to convert [" + op14gl + "] [" + ppl1 + "]");
         }
         final String ppl2 = parseProgressCharLiteral("'" + op24gl + "'", !unixEscapes);
         if (!ppl2.equals(this.op2.toStringMessage()))
         {
            System.out.println("Failed to convert [" + op24gl + "] [" + ppl2 + "]");
         }
         
         this.begins   = begins;
         this.eq       = eq;
         this.ne       = ne;
         this.gt       = gt;
         this.lt       = lt;
         this.ge       = ge;
         this.le       = le;
         this.matches  = matches;
         this.cBegins  = cBegins;
         this.cEq      = cEq;
         this.cNe      = cNe;
         this.cGt      = cGt;
         this.cLt      = cLt;
         this.cGe      = cGe;
         this.cLe      = cLe;
         this.cMatches = cMatches;
      }
      
      boolean test()
      {
         boolean flag = true;
         
         op1.setCaseSensitive(false);
         if (TextOps._begins(op1, op2) != begins)
         {
            flag = false;
            System.out.println("begins failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (TextOps._begins(op1, op2) != cBegins)
         {
            flag = false;
            System.out.println("begins(c) failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (CompareOps.equals(op1, op2) != eq)
         {
            flag = false;
            System.out.println("isEqual failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (CompareOps.equals(op1, op2) != cEq)
         {
            flag = false;
            System.out.println("isEqual(c) failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (CompareOps.notEqual(op1, op2) != ne)
         {
            flag = false;
            System.out.println("notEqual failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (CompareOps.notEqual(op1, op2) != cNe)
         {
            flag = false;
            System.out.println("notEqual(c) failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (CompareOps.greaterThan(op1, op2) != gt)
         {
            flag = false;
            System.out.println("isGreaterThan failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (CompareOps.greaterThan(op1, op2) != cGt)
         {
            flag = false;
            System.out.println("isGreaterThan(c) failure for \"" + op1 +
                                "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (CompareOps.lessThan(op1, op2) != lt)
         {
            flag = false;
            System.out.println("isLessThan failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (CompareOps.lessThan(op1, op2) != cLt)
         {
            flag = false;
            System.out.println("isLessThan(c) failure for \"" + op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (CompareOps.greaterThanOrEqual(op1, op2) != ge)
         {
            flag = false;
            System.out.println("isGreaterThanOrEqual failure for \"" + op1 +
                               "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (CompareOps.greaterThanOrEqual(op1, op2) != cGe)
         {
            flag = false;
            System.out.println("isGreaterThanOrEqual(c) failure for \"" +
                                op1 + "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (CompareOps.lessThanOrEqual(op1, op2) != le)
         {
            flag = false;
            System.out.println("isLessThanOrEqual failure for \"" + op1 +
                               "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(true);
         if (CompareOps.lessThanOrEqual(op1, op2) != cLe)
         {
            flag = false;
            System.out.println("isLessThanOrEqual(c) failure for \"" + op1 +
                                "\" and \"" + op2 + "\"");
         }
         op1.setCaseSensitive(false);
         if (TextOps._matches(op1, op2, !unixEscapes) != matches)
         {
            flag = false;
            System.out.println("matches failure for \"" + this.op1 + 
                               "\" (\"" + op1 + "\") and \"" + this.op2 + 
                               "\" (\"" + op2 + "\")");
         }
         op1.setCaseSensitive(true);
         if (TextOps._matches(op1, op2, !unixEscapes) != cMatches)
         {
            flag = false;
            System.out.println("matches(c) failure for \"" + this.op1 +
                                "\" (\"" + op1 + "\") and \"" + this.op2 +
                                "\" (\"" + op2 + "\")");
         }
         
         return flag;
      }
   }
   
   /**
    * An immutable character type, to be used for constants cached in {@link character#CHARACTER_CONSTANTS}.
    * <p>
    * Instances of this type are serializable.
    */
   public static class characterConstant
   extends character
   implements BaseDataTypeConstant
   {
      /** Pre-calculated hash code. */
      private final int hashCode;
      
      /**
       * Default constructor, creates an instance that represents the unknown value.
       */
      public characterConstant()
      {
         // unknown
         hashCode = 0;
         unknown = true;
         value = null;
      }

      /**
       * Default constructor, creates an instance that for the given value.
       */
      public characterConstant(String value)
      {
         if (value != null)
         {
            value = javaSpacifyNull(value);
         }
         
         this.javaSpacifyNull = false;
         this.unknown = value == null;
         this.value = value;
         this.hashCode = unknown ? 0 : value.hashCode();
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public void deepAssign(Text value)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public void setCaseSensitive(boolean caseSens)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public void setUnknown()
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public void setValue(String value)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      void forceBytes(byte[] data)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      void forceBytes(byte[] data, int length)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public character truncateValue()
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public void assign(BaseDataType value) 
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public void assign(Text value)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Operation not possible.
       * 
       * @throws   UnsupportedOperationException
       *           This is an immutable character instance.
       */
      @Override
      public Text replaceEntry(double i, character delimit, String replace)
      {
         throw new UnsupportedOperationException("This is an immutable character instance.");
      }
      
      /**
       * Get the pre-calculated {@link #hashCode}.
       *
       * @return  Hash code value for this object instance.
       */
      @Override
      public int hashCode()
      {
         return hashCode;
      }
      
      /**
       * Equality test.
       * <p>
       * Only character constants can be compared.  If trying to compare against a {@link character}, an
       * {@link IllegalStateException} will be thrown - constants can't escape the cache, real character 
       * instances must be used.
       * 
       * @param    ref
       *           The instance to test.
       * 
       * @return   See above.
       */
      @Override
      public boolean equals(Object ref)
      {
         if (ref == this)
         {
            return true;
         }
         
         if (!(ref instanceof characterConstant))
         {
            if (ref instanceof character)
            {
               // these instances can't 'escape' outside the cache - they need to be assigned immediately.
               throw new IllegalStateException("Can not compare character with characer constant!");
            }
            
            return false;
         }
         
         characterConstant cc = (characterConstant) ref;
         
         return cc.unknown == this.unknown && cc.value.equals(this.value);
      }
   }
   
   /**
    * Command line interface which provides a test harness to regression
    * test this class.  No parameters are necessary.
    */
   public static void main(String[] args)
   {
      // if still true after all tests, everything passed
      boolean flag = true;
      TestCase.unixEscapes = true; // the TestCase constructor uses it
      TestCase linuxTests[] =
      {
         new TestCase(" hello "    , "hello"      , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("hello"      , "hello  "    , false, true , false, false,
                                                    false, true , true , false,
                                                    false, true , false, false,
                                                    false, true , true , false),
         new TestCase("hello    "  , "hello"      , true , true , false, false,
                                                    false, true , true , false,
                                                    true , true , false, false,
                                                    false, true , true , false),
         new TestCase("hELLo "     , "hello"      , true , true , false, false,
                                                    false, true , true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("hello    "  , "heLLo"      , true , true , false, false,
                                                    false, true , true , false,
                                                    false, false, true , true ,
                                                    false, true , false, false),
         new TestCase(" hello "    , ".ell."      , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" hello "    , ".*ell*"     , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" hello~hel", "..ell.\\*"   , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" hel~lo "    , " hell*"    , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase(" ajaj\\aj " , "*aj*"       , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj\\aj " , "*aj\\aj "   , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj\\aj " , "*ajaj "     , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj\\\\aj " , "*ajaj "   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\\\aj " , "*ajaj " , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\aj " , "*aj\\aj " , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\aj ", "*aj\\\\aj ", false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\aj ", " ajaj\\\\aj ", true , true , false, false,
                                                    false, true , true , false,
                                                    true , true , false, false,
                                                    false, true , true , false),
         new TestCase(" ajaj~aj " , "*aj*"        , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj~aj "   , "*aj~~aj "  , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj~aj "   , "*ajaj "    , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj~~aj "  , "*ajaj "    , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~~aj "  , "*ajaj "   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~aj "  , "*aj~aj "   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~aj "  , "*aj~~aj "  , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~aj "  , " ajaj~~aj ", true , true , false, false,
                                                    false, true , true , false,
                                                    true , true , false, false,
                                                    false, true , true , false),
         new TestCase("a\\a"         , "a~~a"     , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase("a\\a"         , "a\\\\a"   , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("a\\\\a"      , "a\\\\\\\\a", false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("a~~a"        , "a~~~~a"    , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase("*a.b"       , "~~*a~~.b"   , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase("*a.b"       , "~*a~.b"     , true , true , false, false,
                                                    false, true , true , true ,
                                                    true , true , false, false,
                                                    false, true , true , true ),
         new TestCase("*a.b"       , "*a.b"       , true , true , false, false,
                                                    false, true , true , true ,
                                                    true , true , false, false,
                                                    false, true , true , true ),
         new TestCase("*axb"       , "~~*a~~.b"   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("*axb"       , "~*a~.b"     , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("*axb"       , "*a.b"       , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("xa.b"       , "~~*a~~.b"   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("xa.b"       , "~*a~.b"     , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("xa.b"       , "*a.b"       , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true )
      };
      
      TestCase.unixEscapes = false; // the TestCase constructor uses it
      TestCase windowsTests[] =
      {
         new TestCase(" hello "    , "hello"      , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("hello"      , "hello  "    , false, true , false, false,
                                                    false, true , true , false,
                                                    false, true , false, false,
                                                    false, true , true , false),
         new TestCase("hello    "  , "hello"      , true , true , false, false,
                                                    false, true , true , false,
                                                    true , true , false, false,
                                                    false, true , true , false),
         new TestCase("hELLo "     , "hello"      , true , true , false, false,
                                                    false, true , true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("hello    "  , "heLLo"      , true , true , false, false,
                                                    false, true , true , false,
                                                    false, false, true , true ,
                                                    false, true , false, false),
         new TestCase(" hello "    , ".ell."      , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" hello "    , ".*ell*"     , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" hello~hel", "..ell.\\*"   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" hel~lo "    , " hell*"    , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase(" ajaj\\aj " , "*aj*"       , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj\\aj " , "*aj\\aj "   , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj\\aj " , "*ajaj "     , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\aj " , "*ajaj "   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\\\aj " , "*ajaj " , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\aj " , "*aj\\aj " , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj\\\\aj ", "*aj\\\\aj ", false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj\\\\aj ", " ajaj\\\\aj ", true , true , false, false,
                                                    false, true , true , true ,
                                                    true , true , false, false,
                                                    false, true , true , true ),
         new TestCase(" ajaj~aj " , "*aj*"        , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj~aj "   , "*aj~~aj "  , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj~aj "   , "*ajaj "    , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase(" ajaj~~aj "  , "*ajaj "    , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~~aj "  , "*ajaj "   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~aj "  , "*aj~aj "   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~aj "  , "*aj~~aj "  , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase(" ajaj~~aj "  , " ajaj~~aj ", true , true , false, false,
                                                    false, true , true , false,
                                                    true , true , false, false,
                                                    false, true , true , false),
         new TestCase("a\\a"         , "a~~a"     , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("a\\a"         , "a\\\\a"   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , true ,
                                                    false, true , false, false),
         new TestCase("a\\\\a"      , "a\\\\\\\\a", false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , true ,
                                                    false, true , false, false),
         new TestCase("a~~a"        , "a~~~~a"    , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase("*a.b"       , "~~*a~~.b"   , false, false, true , false,
                                                    true , false, true , true ,
                                                    false, false, true , false,
                                                    true , false, true , true ),
         new TestCase("*a.b"       , "~*a~.b"     , true , true , false, false,
                                                    false, true , true , true ,
                                                    true , true , false, false,
                                                    false, true , true , true ),
         new TestCase("*a.b"       , "*a.b"       , true , true , false, false,
                                                    false, true , true , true ,
                                                    true , true , false, false,
                                                    false, true , true , true ),
         new TestCase("*axb"       , "~~*a~~.b"   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("*axb"       , "~*a~.b"     , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("*axb"       , "*a.b"       , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("xa.b"       , "~~*a~~.b"   , false, false, true , false,
                                                    true , false, true , false,
                                                    false, false, true , false,
                                                    true , false, true , false),
         new TestCase("xa.b"       , "~*a~.b"     , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true ),
         new TestCase("xa.b"       , "*a.b"       , false, false, true , true ,
                                                    false, true , false, true ,
                                                    false, false, true , true ,
                                                    false, true , false, true )
      };
      
      // run the tests
      TestCase.unixEscapes = false;
      System.out.println("Unix escapes: " + TestCase.unixEscapes);
      for (TestCase aTest : windowsTests)
      {
         if (!aTest.test())
         {
            flag = false;
         }
      }
      
      TestCase.unixEscapes = true;
      System.out.println("Unix escapes: " + TestCase.unixEscapes);
      for (TestCase aTest : linuxTests)
      {
         if (!aTest.test())
         {
            flag = false;
         }
      }
      
      if (flag)
      {
         System.out.println("All tests passed successfully.");
      }

      System.out.println(TextOps.matchesList("abc,def", "ABc") + " yes");
      System.out.println(TextOps.matchesList("abc,def", "dEF") + " yes");
      System.out.println(TextOps.matchesList("abc,def", "xyz") + " no ");
      System.out.println(TextOps.matchesList("!abc,def", "ABc") + " no");
      System.out.println(TextOps.matchesList("!abc,def", "dEF") + " yes");
      System.out.println(TextOps.matchesList("abc*,def", "ABc") + " yes");
      System.out.println(TextOps.matchesList("ab*,def", "ABc")  + " yes");
      System.out.println(TextOps.matchesList("a*b*c,def", "AxBc") + " yes");
      System.out.println(TextOps.matchesList("a*b*c,def", "AxByc") + " yes");
      System.out.println(TextOps.matchesList("!a*b*c,def", "AxBc") + " no ");
      System.out.println(TextOps.matchesList("!a*b*c,def", "AxByc") + " no ");
      System.out.println(TextOps.matchesList("a!bc,def", "A!Bc") + " yes");
      System.out.println(TextOps.matchesList("abc,d!ef", "d!EF") + " yes");
      System.out.println(TextOps.matchesList(",abc,def", "") + " yes");
      System.out.println(TextOps.matchesList(" ,abc,def", "") + " no ");
      System.out.println(TextOps.matchesList(" ,abc,def", " ") + " yes");
      System.out.println(TextOps.matchesList("abc,def,", "") + " yes");
      System.out.println(TextOps.matchesList("abc,def,", " ") + " no ");
      System.out.println(TextOps.matchesList("abc,def, ", " ") + " yes");
      System.out.println(TextOps.matchesList("", " ") + " no ");
      System.out.println(TextOps.matchesList("", "") + " yes");
      System.out.println(TextOps.matchesList(new character(), " ") + " ?  ");
      System.out.println(TextOps.matchesList(" ", new character()) + " ?  ");
      System.out.println(TextOps.matchesList(new character(), new character()) + " ?  ");
      System.out.println(TextOps.matchesList(new character()) + " exception");

   }
   
   // This is the Progress 4GL program that was used to generate the test
   // case results.  As output it generates the Java statements to initialize
   // the tests[] array in main(), HOWEVER due to the natural conversion of
   // string literals by Progress (on input), the corresponding output
   // strings are missing tildes and backslashes... so you will have to fix
   // up these strings to include the missing data.  In addition, since Java
   // modifies string literals at compile time, you will need to double up
   // every backslash to ensure that the resulting number of backslashes
   // match what the Progress program used.  So if there were 2 backslashes
   // in the Progress version of a string, the Java version will need 4.
   // Once these fixups occur in the main() method, then the tests can be
   // run to determine the level of compatibility between this class and
   // Progress string comparison operations.
   /*   
   
   def var op1  as char extent 38.
   def var op1_ as char extent 38 case-sensitive.
   def var op2  as char extent 38.
   def var res  as logical format "true/false".
   def var i    as int.
   def var str  as char.
   
   op1[1]  = " hello ".     op2[1]  = "hello".
   op1[2]  = "hello".       op2[2]  = "hello  ".
   op1[3]  = "hello    ".   op2[3]  = "hello".    
   op1[4]  = "hELLo ".      op2[4]  = "hello".
   op1[5]  = "hello    ".   op2[5]  = "heLLo".
   op1[6]  = " hello ".     op2[6]  = ".ell.".
   op1[7]  = " hello ".     op2[7]  = ".*ell*".
   op1[8]  = " hello~hel".  op2[8]  = "..ell.\*".
   op1[9]  = " hel~lo ".    op2[9]  = " hell*".
   op1[10] = " ajaj\aj ".   op2[10] = "*aj*".
   op1[11] = " ajaj\aj ".   op2[11] =  "*aj\aj ".
   op1[12] = " ajaj\aj ".   op2[12] =  "*ajaj ".
   op1[13] = " ajaj\\aj ".  op2[13] =  "*ajaj ".
   op1[14] = " ajaj\\\aj ". op2[14] =  "*ajaj ".
   op1[15] = " ajaj\\aj ".  op2[15] =  "*aj\aj ".
   op1[16] = " ajaj\\aj ".  op2[16] =  "*aj\\aj ".
   op1[17] = " ajaj\\aj ".  op2[17] =  " ajaj\\aj ".
   op1[18] = " ajaj~aj ".   op2[18] =  "*aj*".
   op1[19] = " ajaj~aj ".   op2[19] =  "*aj~aj ".
   op1[20] = " ajaj~aj ".   op2[20] =  "*ajaj ".
   op1[21] = " ajaj~~aj ".  op2[21] =  "*ajaj ".
   op1[22] = " ajaj~~~aj ". op2[22] =  "*ajaj ".
   op1[23] = " ajaj~~aj ".  op2[23] =  "*aj~aj ".
   op1[24] = " ajaj~~aj ".  op2[24] =  "*aj~~aj ".
   op1[25] = " ajaj~~aj ".  op2[25] =  " ajaj~~aj ".
   op1[26] = "a\a".         op2[26] =  "a~~a".
   op1[27] = "a\a".         op2[27] =  "a\\a".
   op1[28] = "a\\a".        op2[28] =  "a\\\\a".
   op1[29] = "a~~a".        op2[29] =  "a~~~~a".
   op1[30] = "*a.b".        op2[30] =  "~~*a~~.b".
   op1[31] = "*a.b".        op2[31] =  "~*a~.b".
   op1[32] = "*a.b".        op2[32] =  "*a.b".
   op1[33] = "*axb".        op2[33] =  "~~*a~~.b".
   op1[34] = "*axb".        op2[34] =  "~*a~.b".
   op1[35] = "*axb".        op2[35] =  "*a.b".
   op1[36] = "xa.b".        op2[36] =  "~~*a~~.b".
   op1[37] = "xa.b".        op2[37] =  "~*a~.b".
   op1[38] = "xa.b".        op2[38] =  "*a.b".
   
   output to value("listing.txt").
   
   do i = 1 to 38.
      str = "         new TestCase(\"" + op1[i] + "\"" +
            fill(" ", 11 - length(op1[i])) + ", \"" + op2[i] + "\"" +
            fill(" ", 11 - length(op2[i])) + ", ". 
      put str format "x(52)".
   
      res = op1[i] begins op2[i].
      put res ", ".
      res = op1[i] eq op2[i].
      put res ", ".   
      res = op1[i] ne op2[i].
      put res ", ".
      res = op1[i] gt op2[i].
      put res ",~n".
      put fill(" ", 52) format "x(52)".
      res = op1[i] lt op2[i].
      put res ", ".   
      res = op1[i] ge op2[i].
      put res ", ".   
      res = op1[i] le op2[i].
      put res ", ".   
      res = op1[i] matches op2[i].
      put res ",~n".   
      put fill(" ", 52) format "x(52)".
      
      op1_[i] = op1[i].
      
      res = op1_[i] begins op2[i].
      put res ", ".
      res = op1_[i] eq op2[i].
      put res ", ".   
      res = op1_[i] ne op2[i].
      put res ", ".
      res = op1_[i] gt op2[i].
      put res ",~n".
      put fill(" ", 52) format "x(52)".
      res = op1_[i] lt op2[i].
      put res ", ".   
      res = op1_[i] ge op2[i].
      put res ", ".   
      res = op1_[i] le op2[i].
      put res ", ".   
      res = op1_[i] matches op2[i].
      put res "),~n".
   end.
   
   output close.
   
   */
}