SecurityOps.java

/*
** Module   : SecurityOps.java
** Abstract : Progress 4GL compatible security interface
**
** Copyright (c) 2006-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
** 001 GES 20060331   @25287 First version implementing the USERID builtin function.
** 002 GES 20060331   @25294 Added ENCODE support (not Progress 4GL compatible).
** 003 NVS 20060418   @25585 Added changePassword() convenience method to be called from
**                           converted code.
** 004 GES 20060831   @29063 Make sure encode output only includes chars from the set A-Z and
**                           a-z using a simple table driven base 52 approach.
** 005 GES 20101005          Added stub (non-functional) methods for the USERID(dbname) variant.
** 006 CA  20130129          Added stubs for misc security-related builtins. Implemented
**                           base64-encode/decode.
** 007 EVL 20130225          Adding conversion support for SETUSERID built-in.
** 008 GES 20130523          Minor fixes for the base64-encode/decode.
** 009 VIG 20130123          Implemented base64Decode and base64Encode methods using the own old
**                           Base64.java class. Added new method checkFormat to show an error
**                           while decoding in specific cases like the Progress does.
** 010 OM  20130917          Added _User metadata handling (configured in directory).
** 011 GES 20140117          Added HEX-DECODE and HEX-ENCODE functions.
** 012 GES 20140130          Added missing radix in hex-decode implementation and properly
**                           calculated the byte array size.
** 013 GES 20141031          Added a compatible version of ENCODE, left old version that is used
**                           only when a configuration flag is set. 
** 014 EVL 20160217          Clean up comments from symbols invalid for Solaris to compile.
** 015 ECF 20160315          Stubbed out generateUUID.
** 016 OM  20160706          Disabled condition 12118.
** 017 OM  20180305          USERID function returns unknown value if unknown arg is passed.
** 018 GES 20180426          Implemented the body (basic support) of generateUUID(), added
**                           methods for GUID.
**     GES 20180502          MESSAGE-DIGEST (and MD5-DIGEST, SHA1-DIGEST) basic implementations.
** 019 CA  20181128          Initial support for GET-DB-CLIENT and SET-DB-CLIENT.
**                           Initial support for GENERATE-RANDOM-KEY, ENCRYPT and DECRYPT (only 
**                           AES_CBC algorithm is supported at this time). 
** 020 CA  20181220          Added getDbClient(String).
** 021 CA  20190516          Added missing encrypt(...) signatures.
** 022 IAS 20190517          Added full support for ENCRYPT/DECRYPT, GENERATE-PBE-KEY
**                           GENERATE-PBE-SALT.
** 023 IAS 20190626          Changed SET-DB-CLIENT support
** 024 IAS 20190703          Changed support for SET-DB-CLIENT for non-default domains
** 025 OM  20190709          Optimize imports. Dropped unused statements.
** 026 EVL 20190719          Adding temporary workaround for validateCP() in certain conditions.
** 027 IAS 20190820          Fixed temporary workaround for validateCP().
** 028 IAS 20191015          Rework SET-CLIENT/SET-DB-CLIENT
** 029 IAS 20200112          Added DOMAIN-ACCESS-CODE validation with for sealed CP
** 030 IAS 20200428          Check if SEC_AUTHETICATION_* tables exist before trying to access them.
** 031 GES 20201125          Added OS-USERID as a FWD-specific 4GL extension.
**     CA  20210621          ENCRYPT must return unknown if the text data is empty or unknown.
**     OM  20210916          Cached information on existence of a table in a database to avoid subsequent
**                           identical SQL statements.
**     CA  20211228          The default domain access code is empty string, and not 'fwd-access-code'.
**     IAS 20220111          Changed message level about missing SEC_AUTHENTICATION_* tables to
**                           INFO. 
**     AL2 20220328          Added methods for BDT and proxy checks.
**     IAS 20220404          Fixed 'tableExists' method to not depend on the table name case
**     IAS 20220408          Check if a cipher requires IV.
**     IAS 20220415          Misc. fixes for error reporting.
**     IAS 20220713          Do not lock database on SET-CLIENT. 
**     TJD 20220504          Java 11 compatibility minor changes
**     CA  20220520          Fixed generateRandomKey, to consider the current algorithm in
**                           SECURITY-POLICY:SYMMETRIC-ENCRYPTION-ALGORITHM.
**     IAS 20220415          Misc. fixes for error reporting.
** 032 TT  20230215          GENERATE-RANDOM-KEY now works correctly with the DES and DES3 ciphers
** 033 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
** 034 DDF 20230705          Replace static field initialization with a method called at server bootstrap.
** 035 SB  20240516          Change messageDigest to return a raw with the unknown value if the data
**                           is unknown. Refs #8778.
** 036 RAA 20240513          Before validating the client-principal, check that the context has been reset.
**     RAA 20240710          Added validateClientPrincipalHandle function.
**     RAA 20240715          Clarified the type of database name occurrences.
**     RAA 20240717          When reading domains in multi-tenant mode, read them from TenantManager.
**     RAA 20240808          Added multi-tenancy check.
** 037 CA  20240809          Improved hexEncode, to avoid StringBuilder.
** 038 OM  20240912          Added no-active transaction check when tenant is changed.
** 039 OM  20241001          getUserIdFromDB() returns the user@domain when tenant is active.
** 040 OM  20241128          Multi-tenant runtime support: selected the proper persistence context, eventually
**                           based on [sharedDb] parameter.
** 041 OM  20241204          The [sec_authentication_domain] and [sec_authentication_system] meta tables are
**                           accessed from default persistent database, not from [landlord] database.
** 042 OM  20250227          Local optimizations: avoided multiple context lookups, etc.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util;
import java.nio.charset.*;
import java.security.*;
import java.sql.*;
import java.util.*;
import java.util.logging.*;
import java.util.regex.*;
import java.util.stream.*;
import javax.crypto.*;
import com.goldencode.p2j.*;
import com.goldencode.p2j.directory.*;
// explicitly import Base64 class, as it conflicts with other versions
import com.goldencode.p2j.directory.Base64;
import com.goldencode.p2j.persist.*;
import com.goldencode.p2j.persist.orm.*;
import com.goldencode.p2j.main.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.security.SecurityManager;
import com.goldencode.p2j.util.ClientPrincipal.*;
import com.goldencode.p2j.util.ClientPrincipalResource.*;
import com.goldencode.p2j.util.CryptoUtils.*;
import com.goldencode.p2j.util.logging.*;

/**
 * Provides static utility methods for Progress 4GL compatible security
 * functions.  At this time, the following Progress 4GL functions are handled:
 * <p>
 * <pre>
 * 4GL Function                   Java Equivalent
 * ---------------------------    ---------------------------
 * BASE64-DECODE                  {@link #base64Decode}
 * BASE64-ENCODE                  {@link #base64Encode}
 * ENCODE                         {@link #encode}
 * ENCRYPT                        {@link #encrypt}
 * GENERATE-PBE-KEY               {@link #generatePBEKey}
 * GENERATE-PBE-SALT              {@link #generatePBESalt}
 * GENERATE-UUID                  {@link #generateUUID}
 * GUID                           {@link #convertUniversalUIDToGlobalUID}
 * MD5-DIGEST                     {@link #md5Digest}
 * MESSAGE-DIGEST                 {@link #messageDigest}
 * OS-USERID                      {@link #getOSUserId}
 * SHA1-DIGEST                    {@link #sha1Digest}
 * USERID                         {@link #getUserId}
 * USERID(dbname)                 {@link #getUserIdFromDB}
 * </pre>
 * <p>
 */
public class SecurityOps
{
   /** Constant for blank user. In 4GL the blank user is equivalent of guest user. */
   public static final String BLANK_USER = "";
   
   /** _User table of the database is empty, no access privilege needed. */
   public static final int FREE_ACCESS = 0;
   
   /** Database has userids stored in _user table, but do not enforce them. */
   public static final int BLANK_ALLOWED = 1;
   
   /** Only with valid user/password are allowed to access a database. */
   public static final int STRICT = 2;
   
   /** _User table based authentication domain type */
   public static final String _USER_TABLE_AUTH_SYSTEM = "_oeusertable";

   /** Random for the salt generation */
   private static final Random RND = new SecureRandom();
   
   /** Context-local instance. */
   private static final ContextLocal<WorkArea> local = new ContextLocal<WorkArea>()
   {
      protected WorkArea initialValue()
      {
         return new WorkArea();
      }
   };
   
   /** The default domain access code. */
   private static final String DEFAULT_DOMAIN_ACCESS_CODE = ""; // the default access code is empty string
   
   /** The default classname for _User security handler */
   private static final String DEFAULT_SECURITY_OPS_HANDLER =
      "com.goldencode.p2j.util.SecurityManagerSecurityOps";
   
   /** Logger */
   private static final CentralLogger LOG = CentralLogger.get(SecurityOps.class);
   
   /** Table of all alphabetic characters used to limit the encode output. */
   private static final String ALPHAS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
                                        "abcdefghijklmnopqrstuvwxyz";
   
   /** Progress error msg, can be shown while base64-encode method error. */
   private static final String ERR12118 = "Unable to allocate memory for result from BASE64-ENCODE function";
   
   /** Progress error msg, can be shown while base64-decode method error. */
   private static final String ERR12119 = "Error converting Base64 to RAW";
   
   /** Base64 PAD service symbol */
   private static final char PAD = '=';
   
   /** Initial CRC value. */
   private static final int INITIAL_CRC_VALUE = 0x11;
   
   /** Number of bytes in the encoded output. */
   private static final int OUTPUT_ARRAY_SIZE = 16;
   
   /** Number of iterations in the core encoding loop. */
   private static final int CORE_LOOP_PASSES = 5;
   
   /** Reversed polynomial used with the CRC-16 algorithm. */
   private static final int REVERSED_POLYNOMIAL = 0xA001;
   
   /** Special authentication domain types */
   private static final Set<String> SPECIAL_AUTH_SYSTEMS = Collections.unmodifiableSet(
         new HashSet<String>() 
         {{
           add("_extsso"); 
           add("_oslocal"); 
         }}
   ); 
   
   /** The current _User security handler for emulation of USERID and SETUSERID P4GL functions. */
   private static final CustomSecurityOps customSecurityOps = instantiateUserHandler();
   
   /** Random number generator. */
   private static final Random RAND = new Random(System.currentTimeMillis());
   
   /** Collects information of the existence of some tables for each physical database. */
   private static final Map<String, Boolean> checkedTables = new HashMap<>();
   
   /** Regexp for extracting key kength */
   private static final Pattern KEY_LEN = Pattern.compile("^.*_([0-9]*)$");
   
   /** Regexp for detecting DES/DES3 cipher */
   private static final Pattern DES = Pattern.compile("^DES3?_.*");
   
   /** The SQL name of the {@code _sec-authentication-domain} meta table.*/
   public static final String META_SEC_AUTHENTICATION_DOMAIN = "meta_sec_authentication_domain";
   
   /** The SQL name of the {@code _sec-authentication-system} meta table.*/
   public static final String META_SEC_AUTHENTICATION_SYSTEM = "meta_sec_authentication_system";
   
   /** Flag to enable the old incompatible ENCODE. This is 4GL compatible by default. */
   private static boolean compatible = true;
   
   /** The timestamps when the _sec-auth-system were invalidated by a write operation on that table. */
   private static final Map<String, Long> authSystemInvalidationTime =
         Collections.synchronizedMap(new HashMap<>());
   
   /** The timestamps when the _sec-auth-domain were invalidated by a write operation on that table. */
   private static final Map<String, Long> authDomainInvalidationTime =
         Collections.synchronizedMap(new HashMap<>());
   
   /** The timestamps when the _users were invalidated by a write operation on that table. */
   private static final Map<String, Long> usersInvalidationTime =
         Collections.synchronizedMap(new HashMap<>());
   
   /**
    * Instantiates and returns a SecurityOps handler for emulation of USERID and SETUSERID P4GL
    * functions as configured in the directory.
    * <p>
    * At this moment, only two of implementation are available: SecurityManagerSecurityOps and
    * DatabaseTableSecurityOps. If the directory does not contain an entry for
    * 'userSecurityOpsClass' then the default SecurityManagerSecurityOps will be used.
    *
    * @return  an instance of SecurityOps handler as configured in directory
    */
   private static CustomSecurityOps instantiateUserHandler()
   {
      Directory dir = DirectoryManager.getInstance();
      String handlerName = dir.getString(Directory.ID_RELATIVE,
                                         "userSecurityOpsClass",
                                         DEFAULT_SECURITY_OPS_HANDLER);

      // use the class-loader to get the class into memory
      Class<?> cl = null;
      try
      {
         cl = Class.forName(handlerName);
      }
      catch (Exception exc)
      {
         if (LOG.isLoggable(Level.WARNING))
         {
            LOG.log(Level.WARNING,
                    "Can't load user security handler '" + handlerName + "'. Loading default.",
                    exc);
         }
         return getDefaultUserHandler();
      }

      // attempt to instantiate an object
      Object o = null;
      try
      {
         o = cl.getDeclaredConstructor().newInstance();
      }
      catch (Exception exc)
      {
         if (LOG.isLoggable(Level.WARNING))
         {
            LOG.log(Level.WARNING,
                    "User security handler '" + handlerName + "'cannot be instantiated. Loading default.",
                    exc);
         }
         return getDefaultUserHandler();
      }

      // make sure the object is of the desired type
      if (!(o instanceof CustomSecurityOps))
      {
         if (LOG.isLoggable(Level.WARNING))
         {
            LOG.log(Level.WARNING,
                    "Class '" + handlerName + "'cannot be used as security handler. Loading default.");
         }
         return getDefaultUserHandler();
      }

      return (CustomSecurityOps) o;
   }

   /**
    * Instantiate the default _User handler and returns it. This method must not be called
    * multiple times as only one _User handler should exist.
    *
    * @return  An instance of CustomSecurityOps
    */
   private static CustomSecurityOps getDefaultUserHandler()
   {
      return new SecurityManagerSecurityOps();
   }
   
   /**
    * Method called at server bootstrap that initializes values from the directory
    * configuration. Until this method is called, default values are used.
    */
   public static void bootstrap()
   {
      compatible = Utils.getDirectoryNodeBoolean(null, "encode-compatibility", true, false);
   }
   
   /**
    * Set the timestamp of the last invalidation.
    *
    * @param   database
    *          The database logical name.
    * @param   table
    *          A specific table (legacy OR converted SQL name) to be invalidated.
    */
   public static void invalidate(String database, String table)
   {
      switch (table.toLowerCase())
      {
         case META_SEC_AUTHENTICATION_DOMAIN:          // SQL name
         case TenantManager.SEC_AUTHENTICATION_DOMAIN: // legacy name
            authDomainInvalidationTime.put(database.toLowerCase(), System.currentTimeMillis());
            break;
         case META_SEC_AUTHENTICATION_SYSTEM:          // SQL name
         case TenantManager.SEC_AUTHENTICATION_SYSTEM: // legacy name
            authSystemInvalidationTime.put(database.toLowerCase(), System.currentTimeMillis());
            break;
         case MetadataSecurityOps.META_TABLE_USER: // SQL name
         case "_user":                             // legacy name
            usersInvalidationTime.put(database.toLowerCase(), System.currentTimeMillis());
            break;
      }
   }
   
   /**
    * Returns the userid associated with the current user.
    * <p>
    * Note that in Progress 4GL, this takes an optional string parameter with
    * the name of the logical database in which the userid is set.  There is
    * no corresponding database level authentication concept in P2J so this
    * parameter is not supported.
    *
    * @return   The current user context's userid or the empty string in the
    *           case of any problem.
    */
   public static character getUserId()
   {
      return new character(customSecurityOps.getUserId());
   }
   
   /**
    * Returns the userid associated with the current user of the given logical database.
    * <p>
    * This will call the current configured implementation of CustomSecurityOps. If the database
    * _User metadata is enabled then the authenticated user is returned, otherwise the default
    * SecurityManager user is returned.
    *
    * @param    dbname
    *           Logical database name.
    *
    * @return  The current user context's userid for specified database. If database is not known
    *          or not connected, the empty string is returned. If an {@code unknown} parameter is
    *          provided the method returns {@code unknown} value. 
    */
   public static character getUserIdFromDB(character dbname)
   {
      if (dbname == null || dbname.isUnknown())
      {
         return new character();
      }
      // call the method with unwrapped-argument signature:
      return getUserIdFromDB(dbname.toStringMessage());
   }
   
   /**
    * Returns the userid associated with the current user of the given logical database.
    * <p>
    * This will call the current configured implementation of CustomSecurityOps. If the database
    * _User metadata is enabled then the authenticated user is returned, otherwise the default
    * SecurityManager user is returned.
    *
    * @param    dbname
    *           Logical database name.
    *
    * @return   The current user context's userid or the empty string in the
    *           case of any problem.
    */
   public static character getUserIdFromDB(String dbname)
   {
      ConnectionManager cm = ConnectionManager.get();
      String ldbName = cm.getLDBName(dbname);
      if (ldbName != null && cm.getTenantName(dbname) != null)
      {
         ClientPrincipal cp = ConnectionManager.getDbClient(dbname.toLowerCase());
         if (cp != null)
         {
            return new character(cp.getUserId().toJavaType() + "@" + cp.getDomainName().toJavaType());
         }
      }
      
      // no active tenant call the method with unwrapped-argument signature:
      return new character(customSecurityOps.getUserIdFromDB(dbname));
   }

   /**
    * Returns the os-userid associated with the current user.  This is a FWD-specific 4GL extension which
    * uses native APIs to read the operating-system userid/username (e.g. GetUserNameA in Windows).
    *
    * @return   The user's operating system userid/username or the empty string in the case of any problem.
    */
   public static character getOSUserId() 
   {
      ClientParameters params = StandardServer.getClientParameters();
      return new character(params.osUserName == null ? "" : params.osUserName);
   }
   
   /**
    * Returns the default userid used for authentication to databases. 
    * If configured with metadata security and the database does not have users defined in its
    * metadata, we will use the <code>getDefaultUserid()</code> user id.
    *
    * @param   ldbname
    *          The logical name of the database.
    *
    * @return  the default userid used for authentication to databases.
    */
   public static String getDefaultUserid(String ldbname)
   {
      if (customSecurityOps.getAuthLevel(ldbname) == SecurityOps.FREE_ACCESS)
      {
         return getDefaultUserid();
      }
      else 
      {
         return BLANK_USER;
      }
   }
   
   /**
    * Returns the default userid used for authentication to databases that do not have _User
    * meta-table populated. Usually this is the username from the underlying OS, or empty string
    * if it cannot be detected for various causes.
    *
    * @return  the default userid used for authentication to databases.
    */
   public static String getDefaultUserid()
   {
      try
      {
         // extract the user name from the OS properties
         String defUserid = System.getProperty("user.name");

         // make sure is empty string instead of null
         return defUserid == null ? BLANK_USER : defUserid;
      }
      catch (SecurityException e)
      {
         // the security manager does not allow extracting the information return empty string, too
         return "";
      }
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(String userid, character password, character dbname) 
   {
      return logical.of(customSecurityOps.setUserId(userid,
                                                    password.toStringMessage(),
                                                    dbname.toStringMessage()));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(character userid, String password, character dbname) 
   {
      return logical.of(customSecurityOps.setUserId(userid.toStringMessage(),
                                                    password,
                                                    dbname.toStringMessage()));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(String userid, String password, character dbname) 
   {
      return logical.of(customSecurityOps.setUserId(userid, password, dbname.toStringMessage()));
   }

   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(character userid, character password, String dbname)
   {
      return logical.of(customSecurityOps.setUserId(userid.toStringMessage(),
                                                    password.toStringMessage(),
                                                    dbname));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(String userid, character password, String dbname) 
   {
      return logical.of(customSecurityOps.setUserId(userid, password.toStringMessage(), dbname));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(character userid, String password, String dbname) 
   {
      return logical.of(customSecurityOps.setUserId(userid.toStringMessage(), password, dbname));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(String userid, String password, String dbname) 
   {
      return logical.of(customSecurityOps.setUserId(userid, password, dbname));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(String userid, character password) 
   {
      return logical.of(customSecurityOps.setUserId(userid, password.toStringMessage()));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(character userid, String password) 
   {
      return logical.of(customSecurityOps.setUserId(userid.toStringMessage(), password));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(String userid, String password) 
   {
      return logical.of(customSecurityOps.setUserId(userid, password));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(character userid, character password) 
   {
      return logical.of(customSecurityOps.setUserId(userid.toStringMessage(),
                                                     password.toStringMessage()));
   }
   
   /**
    * Authenticates the user for specified DB connection. Checks if the user login account match
    * the corresponding record of the _User table of the database.
    *
    * @param    userid
    *           The name of the user to set as UserID.
    * @param    password
    *           The user's password.
    * @param    dbname
    *           Logical database name.
    *
    * @return   The <code>true</code> in case of valid user match <code>false</code> otherwise.
    */
   public static logical setUserId(character userid, character password, character dbname)
   {
      return logical.of(customSecurityOps.setUserId(userid.toStringMessage(),
                                                     password.toStringMessage(),
                                                     dbname.toStringMessage()));
   }

   /**
    * Assign the specified CLIENT-PRINCIPAL to all the connected databases.
    * 
    * @param    h
    *           The handle with the CLIENT-PRINCIPAL reference.
    *           
    * @return   <code>true</code> if the USERID was properly set.
    */
   public static logical setDbClient(handle h)
   {
      List<String> ldbNameList = IntStream.range(1, ConnectionManager.getConnectedDbCount())
                                          .mapToObj(i -> ConnectionManager.ldbName(i).getValue())
                                          .collect(Collectors.toList());
      Boolean result = validateClientPrincipalHandle(h, ldbNameList);
      if (result != null)
      {
         return logical.of(result);
      }
      
      ClientPrincipal client = (ClientPrincipal) h.getResource();
      return logical.of(_setDbClient(client, ldbNameList, SealCallee.SET_DB_CLIENT));
   }
   
   /**
    * Assign the specified CLIENT-PRINCIPAL to the specified connected database.
    * 
    * @param    h
    *           The handle with the CLIENT-PRINCIPAL reference.
    * @param    dbNum
    *           The connected database index.
    *           
    * @return   <code>true</code> if the USERID was properly set.
    */
   public static logical setDbClient(handle h, int64 dbNum)
   {
      return dbNum == null || dbNum.isUnknown() ? setDbClient(h, (character) null) 
                                                : setDbClient(h, dbNum.intValue());
   }
   
   /**
    * Assign the specified CLIENT-PRINCIPAL to the specified connected database.
    * 
    * @param    h
    *           The handle with the CLIENT-PRINCIPAL reference.
    * @param    dbNum
    *           The connected database index.
    *           
    * @return   <code>true</code> if the USERID was properly set.
    */
   public static logical setDbClient(handle h, int dbNum)
   {
      String dbName = ConnectionManager.ldbName(dbNum).getValue();
      return setDbClient(h, dbName);
   }
   
   /**
    * Assign the specified CLIENT-PRINCIPAL to the specified connected database.
    * 
    * @param    h
    *           The handle with the CLIENT-PRINCIPAL reference.
    * @param    ldbName
    *           The logical database name or alias.
    *           
    * @return   <code>true</code> if the USERID was properly set.
    */
   public static logical setDbClient(handle h, character ldbName)
   {
      return setDbClient(h, 
            ldbName == null || ldbName.isUnknown() ? null : ldbName.toStringMessage(),
            SealCallee.SET_DB_CLIENT);
   }
   
   /**
    * Assign the specified CLIENT-PRINCIPAL to the specified connected database.
    * 
    * @param    h
    *           The handle with the CLIENT-PRINCIPAL reference.
    * @param    ldbName
    *           The logical database name or alias.
    * @return   <code>true</code> if the USERID was properly set.
    */
   public static logical setDbClient(handle h, String ldbName)
   {
      return setDbClient(h, ldbName, SealCallee.APP);
   }

   /**
    * Assign the specified CLIENT-PRINCIPAL to the specified connected database.
    * 
    * @param    h
    *           The handle with the CLIENT-PRINCIPAL reference.
    * @param    ldbName
    *           The logical database name or alias.
    * @param    callee
    *           Where the method is called from.
    *           
    * @return   <code>true</code> if the USERID was properly set.
    */
   public static logical setDbClient(handle h, String ldbName, SealCallee callee)
   {
      List<String> ldbNameList = Collections.singletonList(ldbName);
      Boolean result = validateClientPrincipalHandle(h, ldbNameList);
      if (result != null)
      {
         return logical.of(result);
      }
      
      ClientPrincipal client = (ClientPrincipal) h.getResource();
      if (TenantConfig.isEnabled())
      {
         if (BufferManager.get().isTransaction())
         {
            int[] errCodes = new int[] {16402, 16367, 13691, 12710};
            String[] errMsgs = new String[] {
                  "Cannot change tenant identity from within a transaction",
                  "CLIENT-PRINCIPAL validation failed in Test_DB because - " + "Insufficient privileges",
                  "Failed to set " + ldbName + "user id.",
                  "Error setting client principal"
            };
            ErrorManager.recordOrShowError(errCodes, errMsgs, false, false, false, false, false);
            return logical.FALSE;
         }
         
         ConnectionManager.putTenantDatabaseConnected(client, ldbName); // uses the 'old' _meta database 
      }
      
      ldbName = validateDbName(ldbName);
      return ldbName == null ? logical.FALSE : logical.of(_setDbClient(client, ldbNameList, callee));
   }

   /**
    * Validate database name.
    * 
    * @param ldbName database name.
    * 
    * @return  The logical name, or null if <code>name</code> is invalid.
    */
   private static String validateDbName(String ldbName)
   {
      if (ldbName == null)
      {
         ErrorManager.recordOrShowError(new int[] {12709}, 
                                        new String[] {"Invalid database identifier in SET-DB-CLIENT"},
                                        false, false, false, false, false);
         return null; 
      }
      
      ldbName = ConnectionManager.ldbName(ldbName).getValue();
      if (ldbName == null)
      {
         int[] nums = { 12704, 12710 };
         String[] texts = 
         {
            "Invalid parameter for SET-DB-CLIENT",
            "Error setting client principal"
         };
         ErrorManager.recordOrShowError(nums, texts, false, false, false, false, false);
         return null;
      }
      
      return ldbName;
   }
   
   /**
    * Fully read the content of {@code _User} table from the database and store it in local data structure.
    * 
    * @param   wa
    *          The contextual target data structure.
    * @param   db
    *          The database name.
    * @param   persistence
    *          The Persistence instance for the database.
    */
   private static void refreshUsers(WorkArea wa, String db, Persistence persistence)
   {
      if (!wa.dbUsers.isEmpty())
      {
         Long lastWrite = usersInvalidationTime.get(db);
         if (lastWrite == null)
         {
            // no invalidation, then we have the up-to-date revision of _user list 
            return;
         }
         
         long lastRead = wa.usersLastRead.get(db); // should not be null since wa.dbUsers is NOT empty
         if (lastRead >= lastWrite)
         {
            // we have the last copy, no need to update 
            return;
         }
         
         // reaching this point means the list of users is deprecated
         wa.dbUsers.clear();
      }
      
      wa.usersLastRead.put(db, System.currentTimeMillis());
      ConnectionManager.readUserTable(persistence, a -> wa.addUser(db, new QUserId(a[0], a[1]), a[2]));
   }

   /**
    * Read domains data from {@code _sec-authentication-domain} and {@code _sec-authentication-system tables}.
    *
    * @param   wa
    *          The contextual target data structure.
    * @param   db
    *          The database name.
    * @param   persistence
    *          The Persistence instance for the database.
    */
   private static void refreshDomains(WorkArea wa, String db, Persistence persistence)
   {
      boolean doRefresh = false;
      
      if (!wa.dbDomains.isEmpty())
      {
         Long lastWrite = authDomainInvalidationTime.get(db);
         if (lastWrite != null)
         {
            long lastRead = wa.domainsLastRead.get(db); // should not be null since wa.dbDomains is NOT empty
            if (lastRead < lastWrite)
            {
               // we do NOT have the last copy, update needed
               doRefresh = true;
            }
         }
      }
      else
      {
         doRefresh = true;
      }
      
      if (!doRefresh)
      {
         if (!wa.dbDomainTypes.isEmpty())
         {
            Long lastWrite = authSystemInvalidationTime.get(db);
            if (lastWrite != null)
            {
               long lastRead = wa.domainTypesLastRead.get(db); // should not be null since wa.dbDomainTypes is NOT empty
               if (lastRead < lastWrite)
               {
                  // we do NOT have the last copy, update needed 
                  doRefresh = true;
               }
            }
         }
         else
         {
            doRefresh = true;
         }
      }
      
      // prevent next attempts to read from the same db
      if (!doRefresh)
      {
         return;
      }
      wa.dbDomains.clear();
      wa.dbDomainTypes.clear();
      wa.domainsLastRead.put(db, System.currentTimeMillis());
      wa.domainTypesLastRead.put(db, System.currentTimeMillis());
      
      try 
      {
         // If multi-tenancy is enabled, then read the domains persistence-less
         if (TenantConfig.isEnabled())
         {
            TenantManager.readDomains(db, wa);
            return;
         }
         
         // Otherwise, in single-tenant mode, read them as usual
         if (!tableExists(persistence, META_SEC_AUTHENTICATION_SYSTEM, Persistence.SHARED_CTX) ||
             !tableExists(persistence, META_SEC_AUTHENTICATION_DOMAIN, Persistence.SHARED_CTX))
         {
            LOG.log(Level.INFO, 
                    "One or both of SEC_AUTHENTICATION_* tables do not exist in the '" + db +
                    "' database. Directory domains will be used");
            return;
         }
         
         // TODO read authentication callback name and related fields
         String queryDomainTypes =
               "SELECT domain_type, domain_type_description " +
               "FROM " + META_SEC_AUTHENTICATION_SYSTEM;
         ConnectionManager.readQueryResults(queryDomainTypes, persistence, rs -> 
         {
            Object[] row = rs.get();
            wa.addDomainType(db, (String) row[0], (String) row[1], true);
         });
         
         String queryDomains =
               "SELECT domain_name, domain_type, domain_description, domain_access_code, domain_enabled, tenant_name " +
               "FROM " + META_SEC_AUTHENTICATION_DOMAIN;
         ConnectionManager.readQueryResults(queryDomains, persistence, rs -> 
         {
            Object[] row = rs.get();
            wa.addDomain(db, (String) row[0], (String) row[1], (String) row[2],
                         SymmetricEncryption.decrypt((String) row[3]), (Boolean) row[4], (String) row[5]);
         });
      } 
      catch (Exception e)
      {
         LOG.log(Level.INFO,"Failed to load domains from _SEC_AUTHENTICATION_*", e);
      }
   }
   
   /**
    * Check is table exists in the database. The check is done at very low level, in database driver. FWD is
    * not responsible for the construction and only manages the result set response locally.
    * <p>
    * In order to improve performance, the result is cached so a subsequent call will be faster for same
    * combination of physical database/table.
    * 
    * @param   persistence
    *          The {@code Persistence} instance for the database.
    * @param   tableName
    *          The table name to be checked.
    * @param   sharedDb
    *          In case of multi-tenant environment: is the buffer shared or private?
    *
    * @return  {@code true} if table exists.
    */
   public static boolean tableExists(Persistence persistence, String tableName, boolean sharedDb) 
   {
      String key = (persistence.getDatabase(sharedDb).getName() + " " + tableName).toLowerCase();
      
      Boolean exists = checkedTables.get(key);
      if (exists != null)
      {
         // cache hit. We encountered the call for this database/table combination
         return exists;
      }
      
      ResultSet rs = null;
      try
      {
         Connection con = persistence.getSession(sharedDb).getConnection();
         rs = con.getMetaData().getTables(null, null, null, new String[]{"TABLE"});
         while (rs.next())
         {
            String tName = rs.getString("TABLE_NAME");
            if (tName != null && tName.equalsIgnoreCase(tableName))
            {
               checkedTables.put(key, true);
               return true;
            }
         }
      } 
      catch (SQLException | PersistenceException e)
      {
         LOG.log(Level.WARNING, "Failed to check the existence of " + tableName, e);
      }
      finally
      {
         if (rs != null)
         {
            try
            {
               rs.close();
            }
            catch (SQLException sqle)
            {
               //ignore me
            }
         }
      }
      
      // the table was not found defined in database meta structures or some error occurred 
      checkedTables.put(key, false);
      return false;
   }
   
   /**
    * Assign the specified CLIENT-PRINCIPAL to the specified connected database.
    * 
    * @param    client
    *           The CLIENT-PRINCIPAL instance.
    * @param    ldbNames
    *           The list of logical databases names or aliases.
    * @param    callee
    *           where the method is called from
    *           
    * @return   <code>true</code> if the USERID was properly set.
    */
   private static boolean _setDbClient(ClientPrincipal client, List<String> ldbNames, SealCallee callee)
   {
      List<String> fails = new ArrayList<>();
      for (String db : ldbNames)
      {
         if (validateCP(client, db, callee))
         {
            customSecurityOps.setUserIdDirect(client.getUserId().getValue(), db, 
                                              callee != SealCallee.SET_CLIENT);
            ConnectionManager.setDbClient(client, db, callee);
         }
         else
         {
            fails.add(db);
         }
      }
      
      if (!fails.isEmpty())
      {
         if (callee != SealCallee.SET_CLIENT)
         {
            ErrorManager.recordOrShowError(new int[] {12710},
                  new String[] {"Error setting client principal"},
                  false, false, false, false, false);
         }
         return false;
      }
      
      return true;
   }
   
   /**
    * Perform the validation of a client principal handle.
    * 
    * @param   h
    *          The handle of the {@code ClientPrincipal} reference.
    * @param   ldbNames
    *          The list of logical databases names or aliases.
    *
    * @return  {@code null} means that the process calling this one can continue,
    *          {@code true} means that the handle is unknown,
    *          {@code false} if an error appears while validating.
    */
   private static Boolean validateClientPrincipalHandle(handle h, List<String> ldbNames)
   {
      if (h.isUnknown())
      {
         ldbNames.forEach(ConnectionManager::unlock);
         return true;
      }
      
      if (!(h.getResource() instanceof ClientPrincipal))
      {
         ErrorManager.recordOrShowError(new int[] {12710},
                                        new String[] {"Error setting client principal"},
                                        false, false, false, false, false);
         return false;
      }
      
      return null;
   }
   
   /**
    * Get a copy of the CLIENT-PRINCIPAL instance associated with the default connected database.
    * 
    * @return   A copy of the CLIENT-PRINCIPAL, or an empty instance if non is associated.
    */
   public static handle getDbClient()
   {
      return getDbClient((character) null);
   }
   
   /**
    * Get a copy of the CLIENT-PRINCIPAL instance associated with the specified database.
    * 
    * @param    ldbName
    *           The logical database name or alias.
    * 
    * @return   A copy of the CLIENT-PRINCIPAL, or an empty instance if non is associated.
    */
   public static handle getDbClient(String ldbName)
   {
      return getDbClient(new character(ldbName));
   }
   
   /**
    * Get a copy of the CLIENT-PRINCIPAL instance associated with the specified database.
    * 
    * @param    ldbName
    *           The logical database name or alias.
    * 
    * @return   A copy of the CLIENT-PRINCIPAL, or an empty instance if non is associated.
    */
   public static handle getDbClient(character ldbName)
   {
      String db = null;
      if (ldbName != null && !ldbName.isUnknown())
      {
         db = ldbName.getValue();
      }
      
      int[] nums = { 15875, 12796 };
      String[] texts = 
      {
         "Invalid or missing database identifier for GET-DB-CLIENT() (15875)",
         "Error getting client principal. (12796)"
      };

      if (db == null)
      {
         if (ConnectionManager.getConnectedDbCount() != 1)
         {
            ErrorManager.recordOrThrowError(nums, texts, false, true);
            return new handle();
         }
         else
         {
            db = ConnectionManager.getDefaultDatabase();
         }
      }

      db = ConnectionManager.ldbName(db).getValue();
      if (db == null)
      {
         ErrorManager.recordOrThrowError(nums, texts, false, true);
         return new handle();
      }
      
      ClientPrincipal client = ConnectionManager.getDbClient(db);
      
      // TODO: if this is the 'default connection identity', then this object can't be used to
      // set the identity for other db connections
      
      // create an empty CLIENT-PRINCIPAL or create a copy of the entire resource
      handle res = new handle();
      res.assign(client == null ? new ClientPrincipal() : client.clone());
      
      return res;
   }
   
   /**
    * Checks if the currently authenticated user has access to a database.
    * <ul>
    *   <li>The user always has access to any database if the <code>SecurityOps</code> is
    *       configured with the default <code>SecurityManagerSecurityOps</code>.
    *   <li>When using <code>MetadataSecurityOps</code>, if the database does not have any users
    *       defined in _user meta table the authentication is not enforced so any user can access
    *       it. Otherwise, the user must have entered a correct combination of userid / password
    *       to get access to respective database.
    * </ul>
    *
    * @param   ldbName
    *          The logical name of the database.
    *          
    * @return  <code>true</code> if access to requested database is permitted for currently
    *          authenticated user.
    */
   public static boolean hasAccessToDatabase(String ldbName)
   {
      return customSecurityOps.hasAccessToDatabase(ldbName);
   }

   /**
    * Check the access level for a database. The response depend on CustomSecurityOps implementation
    * on which this method will forward the message.
    * 
    * @param   ldbname
    *          The logical database name to query. 
    * 
    * @return  The access level to specified database. 
    */
   public static int getAuthLevel(String ldbname)
   {
      return customSecurityOps.getAuthLevel(ldbname);
   }
   
   /**
    * Performs a password change procedure for the currently logged on user.
    * <p>
    * Application has no control over the procedure itself. 
    *
    * @return   failure/success indication
    *           <code>true</code> means 
    */
   public static logical changePassword() 
   {
      SecurityManager sm = SecurityManager.getInstance();
      
      if (sm != null)
      {
         return logical.of(sm.changePassword());
      }
      
      return logical.FALSE;
   }

   /**
    * <code>ENCODE</code> implementation which can be run in compatible mode (the default
    * which uses {@link #_encode} as a worker) or in the incompatible mode using the J2SE
    * MD5 hashing algorithm. The choice of which version to run is made using a flag
    * that is initialized from the directory when the server starts.
    * <p>
    * <b>WARNING: when the incompatible version is used the implementation is NOT compatible
    * with the Progress 4GL algorithm in any way. Any programs relying on comparisons between
    * data encoded by the Progress 4GL ENCODE and the return value from this method will NOT
    * compare as expected.</b>
    * <p>
    * By default, the Progress 4GL compatible <code>ENCODE</code> algorithm is truly compatible
    * and it will operate on stored/migrated values from the original system as expected. The
    * algorithm takes a source byte array (of any size) and generates a 16-byte one-way hash
    * using a 4GL-proprietary approach combined with a variant of CRC-16.  The basic approach is
    * to spread out the source data in an intermediate accumulator array of 16 unsigned bytes
    * and to repeatedly CRC that data and copy the resulting 2-byte CRC into successive elements
    * of this intermediate accumulator, while building the next CRC results on the accumulated
    * CRC value and the recently modified intermediate accumulator.  This set of CRCs is
    * calculated 8 times (since that will generate 16 bytes).  This basic approach is executed 5
    * times in a row. Each byte of the resulting hashed intermediate accumulator array is then
    * translated into one of the 52 possible uppercase or lowercase English alphabetic
    * characters. That translated result is returned.
    * <p>
    * The detailed COMPATIBLE algorithm:
    * <p>
    * <ol>
    *   <li> Initialize storage that can hold at least 2 unsigned bytes to 0x11. This is the
    *        accumulator for the calculated CRC.
    *   <li> Allocate storage that can hold 16 unsigned bytes.  This will be used as an
    *        intermediate output accumulator which will combine the input data and the
    *        calculated CRC data in a manner that will result in each of the 16 bytes having a
    *        value between 0 and 255 inclusive.
    *   <li> Iterate the following steps 5 times:
    *      <ol>
    *         <li>Walk forward through the array of source data provided and XOR each byte into
    *             the intermediate output accumulator (target array), but with the index of the
    *             target matching a backwards walk. Since the source array can be of any length,
    *             the lower index positions in the target may not ever get source data XOR'd.
    *             Source arrays longer than the target array cause wrapping and that means that
    *             the higher index positions in the target array will have data from multiple
    *             source positions XOR'd. See the {@link #sourceToTargetXor} method for details
    *             as well as the significant cryptographic limitations this causes.
    *         <li>Iterate 8 times, doing the following:
    *            <ol>
    *               <li> Call the {@link #crc16} method, passing in both the intermediate output
    *                    accumulator as well as the CRC accumulator.  This is the core CRC
    *                    algorithm and it will calculate the CRC of the current state of the
    *                    intermediate output accumulator while including the initial state of
    *                    the CRC accumulator.  The resulting returned CRC result is assigned
    *                    back to the CRC accumulator.  The CRC-16 algorithm used is not
    *                    cryptographically sound.  It generates a non-uniform distribution of
    *                    hashed data and collisions are not rare.
    *               <li> The least significant byte (byte 0) of the CRC accumulator will be
    *                    directly stored into an index position in the intermediate output
    *                    accumulator. This index position starts at 0 and increments by 2 for
    *                    every iteration of the nearest containing loop.  Since the intermediate
    *                    output accumulator has 16 elements, that is why the containing loop
    *                    iterates only 8 times.  This byte must be treated as an unsigned value.
    *               <li> The next to least significant byte (byte 1) of the CRC accumulator will
    *                    be directly stored into an index position in the intermediate output
    *                    accumulator. This index position starts at 1 and increments by 2 for
    *                    every iteration of the nearest containing loop. This byte must be
    *                    treated as an unsigned value.
    *            </ol>
    *      </ol>
    *   <li> At this point the intermediate output accumulator contains 16 poorly distributed
    *        hashed bytes.  Translate each byte of the intermediate output accumulator into
    *        a byte from the valid output character set.  Valid output characters include all
    *        English alphabetic letters (both uppercase and lowercase), for 52 possible output
    *        characters.  See the {@link #translateToAlpha} method for details on this process
    *        and for the severe limitations of the resulting non-uniform distribution.
    * </ol>
    * <p>
    * The result of this algorithm is NOT cryptographically sound.  It should not be used for
    * secure purposes.  The flaws in this algorithm come from the processing associated with
    * {@link #sourceToTargetXor}, {@link #crc16} and {@link #translateToAlpha}.
    *
    * @param    data
    *           The data to be encoded.  This must not be <code>null</code>.  The string may be
    *           of any length.  The data will be truncated at any contained null characters.
    *
    * @return   A string with 16 encoded bytes where each byte is one of the 52 possible output
    *           characters.  Returns <code>unknown value</code> if the input is
    *           <code>null</code> or <code>unknown value</code>.
    */
   public static character encode(character data)
   {
      if (data == null || data.isUnknown())
      {
         return new character();
      }
      
      return encode(data.toStringMessage());
   }
   
   /**
    * <code>ENCODE</code> implementation which can be run in compatible mode (the default
    * which uses {@link #_encode} as a worker) or in the incompatible mode using the J2SE
    * MD5 hashing algorithm. The choice of which version to run is made using a flag
    * that is initialized from the directory when the server starts.
    * <p>
    * <b>WARNING: when the incompatible version is used the implementation is NOT compatible
    * with the Progress 4GL algorithm in any way. Any programs relying on comparisons between
    * data encoded by the Progress 4GL ENCODE and the return value from this method will NOT
    * compare as expected.</b>
    * <p>
    * By default, the Progress 4GL compatible <code>ENCODE</code> algorithm is truly compatible
    * and it will operate on stored/migrated values from the original system as expected. The
    * algorithm takes a source byte array (of any size) and generates a 16-byte one-way hash
    * using a 4GL-proprietary approach combined with a variant of CRC-16.  The basic approach is
    * to spread out the source data in an intermediate accumulator array of 16 unsigned bytes
    * and to repeatedly CRC that data and copy the resulting 2-byte CRC into successive elements
    * of this intermediate accumulator, while building the next CRC results on the accumulated
    * CRC value and the recently modified intermediate accumulator.  This set of CRCs is
    * calculated 8 times (since that will generate 16 bytes).  This basic approach is executed 5
    * times in a row. Each byte of the resulting hashed intermediate accumulator array is then
    * translated into one of the 52 possible uppercase or lowercase English alphabetic
    * characters. That translated result is returned.
    * <p>
    * The detailed COMPATIBLE algorithm:
    * <p>
    * <ol>
    *   <li> Initialize storage that can hold at least 2 unsigned bytes to 0x11. This is the
    *        accumulator for the calculated CRC.
    *   <li> Allocate storage that can hold 16 unsigned bytes.  This will be used as an
    *        intermediate output accumulator which will combine the input data and the
    *        calculated CRC data in a manner that will result in each of the 16 bytes having a
    *        value between 0 and 255 inclusive.
    *   <li> Iterate the following steps 5 times:
    *      <ol>
    *         <li>Walk forward through the array of source data provided and XOR each byte into
    *             the intermediate output accumulator (target array), but with the index of the
    *             target matching a backwards walk. Since the source array can be of any length,
    *             the lower index positions in the target may not ever get source data XOR'd.
    *             Source arrays longer than the target array cause wrapping and that means that
    *             the higher index positions in the target array will have data from multiple
    *             source positions XOR'd. See the {@link #sourceToTargetXor} method for details
    *             as well as the significant cryptographic limitations this causes.
    *         <li>Iterate 8 times, doing the following:
    *            <ol>
    *               <li> Call the {@link #crc16} method, passing in both the intermediate output
    *                    accumulator as well as the CRC accumulator.  This is the core CRC
    *                    algorithm and it will calculate the CRC of the current state of the
    *                    intermediate output accumulator while including the initial state of
    *                    the CRC accumulator.  The resulting returned CRC result is assigned
    *                    back to the CRC accumulator.  The CRC-16 algorithm used is not
    *                    cryptographically sound.  It generates a non-uniform distribution of
    *                    hashed data and collisions are not rare.
    *               <li> The least significant byte (byte 0) of the CRC accumulator will be
    *                    directly stored into an index position in the intermediate output
    *                    accumulator. This index position starts at 0 and increments by 2 for
    *                    every iteration of the nearest containing loop.  Since the intermediate
    *                    output accumulator has 16 elements, that is why the containing loop
    *                    iterates only 8 times.  This byte must be treated as an unsigned value.
    *               <li> The next to least significant byte (byte 1) of the CRC accumulator will
    *                    be directly stored into an index position in the intermediate output
    *                    accumulator. This index position starts at 1 and increments by 2 for
    *                    every iteration of the nearest containing loop. This byte must be
    *                    treated as an unsigned value.
    *            </ol>
    *      </ol>
    *   <li> At this point the intermediate output accumulator contains 16 poorly distributed
    *        hashed bytes.  Translate each byte of the intermediate output accumulator into
    *        a byte from the valid output character set.  Valid output characters include all
    *        English alphabetic letters (both uppercase and lowercase), for 52 possible output
    *        characters.  See the {@link #translateToAlpha} method for details on this process
    *        and for the severe limitations of the resulting non-uniform distribution.
    * </ol>
    * <p>
    * The result of this algorithm is NOT cryptographically sound.  It should not be used for
    * secure purposes.  The flaws in this algorithm come from the processing associated with
    * {@link #sourceToTargetXor}, {@link #crc16} and {@link #translateToAlpha}.
    *
    * @param    data
    *           The data to be encoded.  This must not be <code>null</code>.  The string may be
    *           of any length.  The data will be truncated at any contained null characters.
    *
    * @return   A string with 16 encoded bytes where each byte is one of the 52 possible output
    *           characters.  Returns <code>unknown value</code> if the input is
    *           <code>null</code>.
    */
   public static character encode(String data)
   {
      if (data == null)
         return new character();
      
      return new character(compatible ? _encode(data) : _incompatible_encode(data));
   }

   /**
    * Progress 4GL compatible <code>ENCODE</code> algorithm which takes a source byte array
    * (of any size) and generates a 16-byte one-way hash using a proprietary approach
    * combined with a variant of CRC-16.  The basic approach is to spread out the source data
    * in an intermediate accumulator array of 16 unsigned bytes and to repeatedly CRC that data
    * and copy the resulting 2-byte CRC into successive elements of this intermediate
    * accumulator, while building the next CRC results on the accumulated CRC value and the
    * recently modified intermediate accumulator.  This set of CRCs is calculated 8 times
    * (since that will generate 16 bytes).  This basic approach is executed 5 times in a row.
    * Each byte of the resulting hashed intermediate accumulator array is then translated into
    * one of the 52 possible uppercase or lowercase English alphabetic characters. That
    * translated result is returned.
    * <p>
    * The detailed algorithm:
    * <p>
    * <ol>
    *   <li> Initialize storage that can hold at least 2 unsigned bytes to 0x11. This is the
    *        accumulator for the calculated CRC.
    *   <li> Allocate storage that can hold 16 unsigned bytes.  This will be used as an
    *        intermediate output accumulator which will combine the input data and the
    *        calculated CRC data in a manner that will result in each of the 16 bytes having a
    *        value between 0 and 255 inclusive.
    *   <li> Iterate the following steps 5 times:
    *      <ol>
    *         <li>Walk forward through the array of source data provided and XOR each byte into
    *             the intermediate output accumulator (target array), but with the index of the
    *             target matching a backwards walk. Since the source array can be of any length,
    *             the lower index positions in the target may not ever get source data XOR'd.
    *             Source arrays longer than the target array cause wrapping and that means that
    *             the higher index positions in the target array will have data from multiple
    *             source positions XOR'd. See the {@link #sourceToTargetXor} method for details
    *             as well as the significant cryptographic limitations this causes.
    *         <li>Iterate 8 times, doing the following:
    *            <ol>
    *               <li> Call the {@link #crc16} method, passing in both the intermediate output
    *                    accumulator as well as the CRC accumulator.  This is the core CRC
    *                    algorithm and it will calculate the CRC of the current state of the
    *                    intermediate output accumulator while including the initial state of
    *                    the CRC accumulator.  The resulting returned CRC result is assigned
    *                    back to the CRC accumulator.  The CRC-16 algorithm used is not
    *                    cryptographically sound.  It generates a non-uniform distribution of
    *                    hashed data and collisions are not rare.
    *               <li> The least significant byte (byte 0) of the CRC accumulator will be
    *                    directly stored into an index position in the intermediate output
    *                    accumulator. This index position starts at 0 and increments by 2 for
    *                    every iteration of the nearest containing loop.  Since the intermediate
    *                    output accumulator has 16 elements, that is why the containing loop
    *                    iterates only 8 times.  This byte must be treated as an unsigned value.
    *               <li> The next to least significant byte (byte 1) of the CRC accumulator will
    *                    be directly stored into an index position in the intermediate output
    *                    accumulator. This index position starts at 1 and increments by 2 for
    *                    every iteration of the nearest containing loop. This byte must be
    *                    treated as an unsigned value.
    *            </ol>
    *      </ol>
    *   <li> At this point the intermediate output accumulator contains 16 poorly distributed
    *        hashed bytes.  Translate each byte of the intermediate output accumulator into
    *        a byte from the valid output character set.  Valid output characters include all
    *        English alphabetic letters (both uppercase and lowercase), for 52 possible output
    *        characters.  See the {@link #translateToAlpha} method for details on this process
    *        and for the severe limitations of the resulting non-uniform distribution.
    * </ol>
    * <p>
    * The result of this algorithm is NOT cryptographically sound.  It should not be used for
    * secure purposes.  The flaws in this algorithm come from the processing associated with
    * {@link #sourceToTargetXor}, {@link #crc16} and {@link #translateToAlpha}.
    *
    * @param    input
    *           The data to be encoded.  This must not be <code>null</code>.  The string may be
    *           of any length.  The data will be truncated at any contained null characters.
    *
    * @return   A string with 16 encoded bytes where each byte is one of the 52 possible output
    *           characters.
    */
   public static String _encode(String input)
   {
      // the input data is truncated at the first encountered null character
      input = Text.javaTruncateNull(input);
      
      byte[] bin  = input.getBytes();
      int[]  data = new int[bin.length];
      
      // copy the input data into a array large enough to handle unsigned characters
      for (int n = 0; n < bin.length; n++)
      {
         data[n] = bin[n];
      }
      
      short[] storage = new short[OUTPUT_ARRAY_SIZE];
      
      int crc    = INITIAL_CRC_VALUE;
      int passes = CORE_LOOP_PASSES;
      
      while (passes != 0)
      {
         sourceToTargetXor(data, storage);
         
         int idx = 0;
         
         while (idx < storage.length)
         {
            crc = crc16(storage, crc);
            
            storage[idx++] = (byte)(crc & 0xFF);
            storage[idx++] = (byte)((crc >> 8) & 0xFF);
         }
         
         passes--;
      }
      
      return new String(translateToAlpha(storage));
   }
   
   /**
    * Converts binary data into a Base64 encoded text.
    *
    * @param    data
    *           The data to be encoded.
    *
    * @return   The encoded string.  <code>unknown</code> will be returned if the given data is
    *           <code>unknown</code> OR if the given data is initialized but the runtime has no
    *           knowledge of the buffer's size.
    */
   public static longchar base64Encode(BinaryData data)
   {
      if (data == null || data.isUnknown())
      {
         return new longchar();
      }

      if (data instanceof memptr)
      {
         return base64Encode((memptr)data);
      }

      return base64Encode(data.getByteArray());
   }
   
   /**
    * Converts binary data into a Base64 encoded text.
    *
    * @param    data
    *           The data to be encoded.
    *
    * @return   The encoded string.  <code>unknown</code> will be returned if the given data is
    *           <code>null</code>.
    */
   public static longchar base64Encode(byte[] data)
   {
      if (data == null)
      {
         return new longchar();
      }

      return new longchar(Base64.byteArrayToBase64(data));
   }

   /**
    * Decodes a base64 encoded string into a binary value.
    *
    * @param    data
    *           The string to be decoded.
    *
    * @return   a memptr with the decoded data.
    */
   public static memptr base64Decode(Text data)
   {
      if (data == null || data.isUnknown()) 
      {
         return new memptr();
      }

      return base64Decode(data.getValue());
   }

   /**
    * Converts a base64 encoded string into a binary value and returns a 
    * memptr with the decoded value. Shows an error 12119 if the source
    * data do not conforms to special format (that Progress checks).
    *
    * @param    data
    *           The string to be decoded.
    *
    * @return   a memptr with the decoded data.
    */
   public static memptr base64Decode(String data)
   {
      if (data == null)
      {
         return new memptr();
      }

      if (!checkFormat(data))
      {
         ErrorManager.recordOrShowError(12119, ERR12119, false, false, false);
         return new memptr();
      }

      // Apache commons Base64.decodeBase64() is incompatible
      return new memptr(Base64.base64ToByteArray(data));
   }
   
   /**
    * Converts a hexadecimal encoded string into a binary value and returns a 
    * raw with the decoded value.  The data must contain an even number of valid
    * hexadecimal digits.
    *
    * @param    data
    *           The text to be decoded.  <code>null</code> will be treated like
    *           unknown value.
    *
    * @return   The decoded binary data if the input is an even number of valid
    *           hexadecimal digits, otherwise it will return unknown value. The
    *           exception is for an empty string, which will return a 0-sized
    *           buffer.
    */
   public static raw hexDecode(Text data)
   {
      if (data == null || data.isUnknown()) 
      {
         return raw.instantiateUnknownRaw();
      }
      
      return hexDecode(data.toStringMessage());
   }
   
   /**
    * Converts a hexadecimal encoded string into a binary value and returns a 
    * raw with the decoded value.  The data must contain an even number of valid
    * hexadecimal digits.
    *
    * @param    data
    *           The string to be decoded.  <code>null</code> will be treated like
    *           unknown value.
    *
    * @return   The decoded binary data if the input is an even number of valid
    *           hexadecimal digits, otherwise it will return unknown value. The
    *           exception is for an empty string, which will return a 0-sized
    *           buffer.
    */
   public static raw hexDecode(String data)
   {
      if (data == null || (data.length() % 2) != 0)
      {
         return raw.instantiateUnknownRaw();
      }
      
      // TODO: this was written from the Progress docs and has only been lightly
      //       tested; the boundary conditions need to be tested to confirm if this
      //       really is correct

      int    len     = data.length();
      byte[] decoded = new byte[len / 2];
      
      for (int i = 0; i < len; i += 2)
      {
         int next = Integer.parseInt(data.substring(i, i + 2), 16);
         
         decoded[i / 2] = (byte) (next & 0xFF);
      }
      
      return new raw(decoded);
   }
   
   /**
    * Converts a binary value into a hexadecimal encoded string and returns a 
    * the encoded value.  The data will result an even number of valid
    * hexadecimal digits.
    *
    * @param    data
    *           The text to be encoded.  <code>null</code> will be treated like
    *           unknown value.
    *
    * @return   The hexadecimal encoded text if the input is not unknown value, or the
    *           unknown value if that was the input. An empty binary buffer will return
    *           an empty string.
    */
   public static character hexEncode(BinaryData data)
   {
      if (data == null || data.isUnknown()) 
      {
         return new character();
      }
      
      String res = hexEncodeImpl(data.getByteArray());
      return new character(res);
   }
   
   /**
    * Encrypts the source and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    *           
    * @return   See above.
    */
   public static memptr encrypt(String data)
   {
      return encrypt(data, SecurityPolicyManager.getSymmetricEncryptionKeyInternal());
   }
   
   /**
    * Encrypts the source using the given key and returns a {@link memptr} instance containing the 
    * encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    *           
    * @return   See above.
    */
   public static memptr encrypt(String data, raw key)
   {
      return encrypt(data, key, SecurityPolicyManager.getSymmetricEncryptionIV());
   }
   
   /**
    * Encrypts the source using the given key and initialization vector, and returns a 
    * {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    *           
    * @return   See above.
    */
   public static memptr encrypt(String data, raw key, raw ivValue)
   {
      return encrypt(data, key, ivValue, SecurityPolicyManager.getSymmetricEncryptionAlgorithm());
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(String data,
                                raw    key,
                                raw    ivValue,
                                String algorithm)
   {
      return encrypt(data, 
                     key, 
                     ivValue, 
                     new character(algorithm));
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(String    data,
                                raw       key,
                                raw       ivValue,
                                character algorithm)
   {
      return encrypt(new character(data), key, ivValue, algorithm);
   }
   
   /**
    * Encrypts the source and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    *           
    * @return   See above.
    */
   public static memptr encrypt(Text data)
   {
      return encrypt(data, SecurityPolicyManager.getSymmetricEncryptionKeyInternal());
   }
   
   /**
    * Encrypts the source using the given key and returns a {@link memptr} instance containing the 
    * encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    *           
    * @return   See above.
    */
   public static memptr encrypt(Text data, raw key)
   {
      return encrypt(data, key, SecurityPolicyManager.getSymmetricEncryptionIV());
   }
   
   /**
    * Encrypts the source using the given key and initialization vector, and returns a 
    * {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    *           
    * @return   See above.
    */
   public static memptr encrypt(Text data, raw key, raw ivValue)
   {
      return encrypt(data, key, ivValue, SecurityPolicyManager.getSymmetricEncryptionAlgorithm());
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(Text data,
                                raw    key,
                                raw    ivValue,
                                String algorithm)
   {
      return encrypt(data,
                     key,
                     ivValue,
                     new character(algorithm));
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(Text      data,
                                raw       key,
                                raw       ivValue,
                                character algorithm)
   {
      if (data.isUnknown() || data.getValue().isEmpty())
      {
         // can't encrypt an unknown or zero-length data
         return new memptr();
      }
      
      raw r = new raw();
      String value = data.getValue();
      r.setString(value, 1, value == null ? 0 : value.length());

      return encrypt(r, key, ivValue, algorithm);
   }
   
   /**

    * 
    * @param    data
    *           The data to be encrypted.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(BinaryData data)
   {
      byte[] bdata = encryptDecryptWorker(Cipher.ENCRYPT_MODE, data, 
            SecurityPolicyManager.getSymmetricEncryptionKeyInternal(), 
            SecurityPolicyManager.getSymmetricEncryptionIV(), 
            SecurityPolicyManager.getSymmetricEncryptionAlgorithm()
      );
      
      return new memptr(bdata);
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(BinaryData data, raw key)
   {
      byte[] bdata = encryptDecryptWorker(Cipher.ENCRYPT_MODE, data, key, 
            SecurityPolicyManager.getSymmetricEncryptionIV(), 
            SecurityPolicyManager.getSymmetricEncryptionAlgorithm()
      );
      
      return new memptr(bdata);
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(BinaryData data,
                                raw        key,
                                raw        ivValue)
   {
      byte[] bdata = encryptDecryptWorker(Cipher.ENCRYPT_MODE, data, key, ivValue, 
            SecurityPolicyManager.getSymmetricEncryptionAlgorithm()
      );
      
      return new memptr(bdata);
   }
   
   /**
    * Encrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the encrypted data.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr encrypt(BinaryData data,
                                raw        key,
                                raw        ivValue,
                                character  algorithm)
   {
      byte[] bdata = encryptDecryptWorker(Cipher.ENCRYPT_MODE, data, 
               key == null || key.isUnknown() ? 
                     SecurityPolicyManager.getSymmetricEncryptionKeyInternal() : key, 
               ivValue == null || ivValue.isUnknown() ?
                     SecurityPolicyManager.getSymmetricEncryptionIV() : ivValue, 
               algorithm == null || algorithm.isUnknown() ?
                     SecurityPolicyManager.getSymmetricEncryptionAlgorithm() : algorithm
            );
      
      return new memptr(bdata);
   }
   
   /**
    * Decrypts the source and returns a {@link memptr} instance containing the decrypted data.
    * 
    * @param    data
    *           The data to be decrypted.
    *           
    * @return   See above.
    */
   public static memptr decrypt(BinaryData data)
   {
      return decrypt(data, 
            SecurityPolicyManager.getSymmetricEncryptionKeyInternal(), 
            SecurityPolicyManager.getSymmetricEncryptionIV(), 
            SecurityPolicyManager.getSymmetricEncryptionAlgorithm()
      );
   }

   /**
    * Decrypts the source using the given key and returns a {@link memptr} instance containing the 
    * decrypted data.
    * 
    * @param    data
    *           The data to be decrypted.
    * @param    key
    *           The decryption key.
    *           
    * @return   See above.
    */
   public static memptr decrypt(BinaryData data, raw key)
   {
      return decrypt(data, key, 
            SecurityPolicyManager.getSymmetricEncryptionIV(), 
            SecurityPolicyManager.getSymmetricEncryptionAlgorithm()
      );
   }

   /**
    * Decrypts the source using the given key and initialization vector, and returns a 
    * {@link memptr} instance containing the decrypted data.
    * 
    * @param    data
    *           The data to be decrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    *           
    * @return   See above.
    */
   public static memptr decrypt(BinaryData data, raw key, raw ivValue)
   {
      return decrypt(data, key, ivValue, SecurityPolicyManager.getSymmetricEncryptionAlgorithm());
   }

   /**
    * Decrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the decrypted data.
    * 
    * @param    data
    *           The data to be decrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr decrypt(BinaryData data,
                                raw        key,
                                raw        ivValue,
                                String     algorithm)
   {
      return decrypt(data, key, ivValue, new character(algorithm));
   }
   
   /**
    * Decrypts the source using the given key, initialization vector and symmetric encryption 
    * algorithm, and returns a {@link memptr} instance containing the decrypted data.
    * 
    * @param    data
    *           The data to be decrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   public static memptr decrypt(BinaryData data,
                                raw        key,
                                raw        ivValue,
                                character  algorithm)
   {
      byte[] bdata = encryptDecryptWorker(Cipher.DECRYPT_MODE, data, 
               key == null || key.isUnknown() ? 
                     SecurityPolicyManager.getSymmetricEncryptionKeyInternal() : key, 
               ivValue == null || ivValue.isUnknown() ?
                     SecurityPolicyManager.getSymmetricEncryptionIV() : ivValue, 
               algorithm == null || algorithm.isUnknown() ?
                     SecurityPolicyManager.getSymmetricEncryptionAlgorithm() : algorithm
            );
      
      return new memptr(bdata);
   }

   /**
    * Generates a random byte-array of the length required by the algorithm specified via 
    * SECURITY-POLICY:SYMMETRIC-ENCRYPTION-ALGORITHM.  If this is not specified, then the default
    * AES_CBC_128 algorithm is used, and 16-byte key is generated.
    * 
    * @return   The random key. 
    * 
    */
   public static raw generateRandomKey()
   {
      // generates random byte-array, with its length dependent on the algorithm in use
      // specified via SECURITY-POLICY:SYMMETRIC-ENCRYPTION-ALGORITHM or the default, AES_CBC_128

      // the algorithm is split in "<alg-name>_<alg-mode>_<key-length>"
      
      int byteLen = SecurityPolicyManager.getSymmetricCiperParams().getKeySize();
      raw r = new raw();
      byte[] bytes = new byte[byteLen];
      RAND.nextBytes(bytes);
      r.setLength(byteLen);
      for (int i = 0; i < bytes.length; i++)
      {
         r.setByte(bytes[i], i + 1);
      }
      
      return r;
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(String password)
   {
      return generatePBEKey(new character(password), null);
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    * @param    salt
    *           a random series of 8 bytes to use as salt value in generating 
    *           the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(String password, raw salt)
   {
      return generatePBEKey(new character(password), salt);
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(Text password)
   {
      return generatePBEKey(password, null);
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    * @param    salt
    *           a random series of 8 bytes to use as salt value in generating 
    *           the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(Text password, raw salt)
   {
      if (password == null || password.isUnknown())
      {
         return raw.instantiateUnknownRaw();
      }
      return generatePBEKey(password.getValue().getBytes(StandardCharsets.UTF_8), salt);
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(BinaryData password)
   {
      return generatePBEKey(password, SecurityPolicyManager.getEncryptionSalt());
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    * @param    salt
    *           a random series of 8 bytes to use as salt value in generating 
    *           the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(BinaryData password, raw salt)
   {
      if (password == null || password.isUnknown())
      {
         return raw.instantiateUnknownRaw();
      }
      return generatePBEKey(password.getByteArray(), salt);
   }
   
   /**
    * Generates a password-based encryption key, based on the PKCS#5/RFC 2898
    * standard, and returns the key as a {@link raw} instance.
    * 
    * @param    password
    *           The password to use in generating the encryption key.
    * @param    salt
    *           a random series of 8 bytes to use as salt value in generating 
    *           the encryption key.
    *           
    * @return   See above.
    */
   public static raw generatePBEKey(byte[] password, raw salt)
   {
      if (salt == null || salt.isUnknown())
      {
         salt = SecurityPolicyManager.getEncryptionSalt();
      }
      try
      {
         byte[] pbe = CryptoUtils.generatePbeKey(
               password, 
               salt.getByteArray(), 
               SecurityPolicyManager.getPbeHashAlgorithm().toStringMessage(), 
               SecurityPolicyManager.getPbeKeyRounds().intValue()
         );
         return new raw(pbe);
      } 
      catch (Exception e)
      {
         ErrorManager.recordOrThrowError(-1, "FWD: generatePbeKey failed " + 
               e.getMessage(), true);
         return null;
      }
   }
   /**
    * Generates a random salt value (a series of 8 bytes) to use in generating
    * an encryption key, and returns the salt value as a {@link raw} instance.
    * 
    * @return   See above.
    */
   public static raw generatePBESalt()
   {
      byte[] salt = new byte[8];
      RND.nextBytes(salt);
      return new raw(salt);
   }
   
   /**
    * Generate a random 16-byte UUID.
    * 
    * @return  UUID.
    */
   public static raw generateUUID()
   {
      // TODO: this is just a basic implementation, there may be compatibility issues
      //       especially if the 4GL uses a UUID version that is different (not the
      //       "random" version 4 that we use here) OR if the cryptographic support
      //       in the 4GL appreciable differs from Java OR if there is some kind of
      //       other edge conditions/error handling that is not matched
      
      raw result = new raw();
      
      // we use a "type 4" random UUID here, which seems different from the 4GL version
      // it will probably work for virtually any use case except a scenario that hard
      // codes to some idiosyncracy of the 4GL output
      UUID uuid = UUID.randomUUID();
      
      result.setInt64(uuid.getMostSignificantBits(), 1);
      result.setInt64(uuid.getLeastSignificantBits(), 9);
      
      return result;
   }
   
   /**
    * Generate a new 16-byte UUID (universally unique id) and return it as a text based 
    * hexadecimal representation that matches a GUID (globally unique id) hyphen separated
    * format.
    *
    * @return   The GUID formatted UUID as text.
    */
   public static character convertUniversalUIDToGlobalUID()
   {
      return convertUniversalUIDToGlobalUID(generateUUID());
   }
   
   /**
    * Convert a 16-byte UUID (universally unique id) into a text based hexadecimal representation
    * that matches a GUID (globally unique id) hyphen separated format.
    *
    * @param    uuid
    *           The 16-byte data to convert.
    *
    * @return   The GUID formatted UUID as text.
    */
   public static character convertUniversalUIDToGlobalUID(raw uuid)
   {
      // TODO: this is just a basic implementation based on simple tests and the 4GL
      //       docs; it is possibly incompatible at least in regard to error handling
      
      if (uuid == null || uuid.isUnknown())
      {
         return new character();
      }
      
      if (uuid.lengthOf() != 16)
      {
         ErrorManager.recordOrThrowError(12708,
                                         "Wrong length UUID detected in GUID function",
                                         false);
         return new character();
      }
      
      String hex = hexEncodeImpl(uuid.getByteArray());
      
      // split into the canonical 8-4-4-4-12 format
      String result = hex.substring(0, 8)   +
                      "-"                   +
                      hex.substring(8, 12)  +
                      "-"                   +
                      hex.substring(12, 16) +
                      "-"                   +
                      hex.substring(16, 20) +
                      "-"                   +
                      hex.substring(20);
      
      return new character(result);
   }
      
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(String data)
   {
      return messageDigest("SHA-1", data, (String) null);
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(String data, String key)
   {
      return messageDigest("SHA-1", data, new character(key));
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(String data, Text key)
   {
      return messageDigest("SHA-1", new character(data), key);
   }

   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(String data, BinaryData key)
   {
      return messageDigest("SHA-1", new character(data), key);
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(Text data)
   {
      return messageDigest("SHA-1", data, (String) null);
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(Text data, String key)
   {
      return messageDigest("SHA-1", data, new character(key));
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(Text data, Text key)
   {
      return messageDigest("SHA-1", data, key);
   }

   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(Text data, BinaryData key)
   {
      return messageDigest("SHA-1", data, key);
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(BinaryData data)
   {
      return messageDigest("SHA-1", data, (String) null);
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(BinaryData data, String key)
   {
      return messageDigest("SHA-1", data, new character(key));
   }
   
   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(BinaryData data, Text key)
   {
      return messageDigest("SHA-1", data, key);
   }

   /**
    * Hashes the specified data using the SHA-1 algorithm, and returns a 
    * 20-byte binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw sha1Digest(BinaryData data, BinaryData key)
   {
      return messageDigest("SHA-1", data, key);
   }

   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(String data)
   {
      return messageDigest("MD5", data, (String) null);
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(String data, String key)
   {
      return messageDigest("MD5", data, new character(key));
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(String data, Text key)
   {
      return messageDigest("MD5", new character(data), key);
   }

   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(String data, BinaryData key)
   {
      return messageDigest("MD5", new character(data), key);
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(String data, BaseDataType key)
   {
      // the value may be a proxy; make sure to unwrap and work with the real value
      key = key == null ? null : key.val();
      if (key instanceof Text) 
      {
         return messageDigest("MD5", data, (Text) key);
      } 
      else if (key instanceof BinaryData)
      {
         return messageDigest("MD5", data, (BinaryData) key);
      }
      else
      {
         ErrorManager.recordOrThrowError(
                  new NumberedException("Unacceptable datatype for MD5-DIGEST argument", 12117));
         return null;
      }
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(Text data)
   {
      return messageDigest("MD5", data, (String) null);
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(Text data, String key)
   {
      return messageDigest("MD5", data, new character(key));
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(Text data, Text key)
   {
      return messageDigest("MD5", data, key);
   }

   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(Text data, BinaryData key)
   {
      return messageDigest("MD5", data, key);
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(Text data, BaseDataType key)
   {
      // the value may be a proxy; make sure to unwrap and work with the real value
      key = key == null ? null : key.val();
      if (key instanceof Text) 
      {
         return messageDigest("MD5", data, (Text) key);
      } 
      else if (key instanceof BinaryData)
      {
         return messageDigest("MD5", data, (BinaryData) key);
      }
      else
      {
         ErrorManager.recordOrThrowError(
                  new NumberedException("Unacceptable datatype for MD5-DIGEST argument", 12117));
         return null;
      }
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(BinaryData data)
   {
      return messageDigest("MD5", data, (String) null);
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(BinaryData data, String key)
   {
      return messageDigest("MD5", data, new character(key));
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(BinaryData data, Text key)
   {
      return messageDigest("MD5", data, key);
   }

   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(BinaryData data, BinaryData key)
   {
      return messageDigest("MD5", data, key);
   }
   
   /**
    * Hashes the specified data using RSA MD5 algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw md5Digest(BinaryData data, BaseDataType key)
   {
      // the value may be a proxy; make sure to unwrap and work with the real value
      key = key == null ? null : key.val();
      if (key instanceof Text) 
      {
         return messageDigest("MD5", data, (Text) key);
      } 
      else if (key instanceof BinaryData)
      {
         return messageDigest("MD5", data, (BinaryData) key);
      }
      else
      {
         ErrorManager.recordOrThrowError(
                  new NumberedException("Unacceptable datatype for MD5-DIGEST argument", 12117));
         return null;
      }
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    *
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, String data)
   {
      return messageDigest(new character(algo), data, (String) null);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, String data, String key)
   {
      return messageDigest(new character(algo), data, new character(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, String data, Text key)
   {
      return messageDigest(new character(algo), new character(data), key);
   }

   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, String data, BinaryData key)
   {
      return messageDigest(new character(algo), new character(data), key);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, Text data)
   {
      return messageDigest(new character(algo), data, (String) null);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, Text data, String key)
   {
      return messageDigest(new character(algo), data, new character(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, Text data, Text key)
   {
      return messageDigest(new character(algo), data, key);
   }

   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, Text data, BinaryData key)
   {
      return messageDigest(new character(algo), data, key);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, BinaryData data)
   {
      return messageDigest(new character(algo), data, (String) null);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, BinaryData data, String key)
   {
      return messageDigest(new character(algo), data, new character(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, BinaryData data, Text key)
   {
      return messageDigest(new character(algo), data, key);
   }

   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(String algo, BinaryData data, BinaryData key)
   {
      return messageDigest(new character(algo), data, key);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, String data)
   {
      return messageDigest(algo, data, (String) null);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, String data, String key)
   {
      return messageDigest(algo, data, new character(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, String data, Text key)
   {
      return messageDigest(algo, new character(data), key);
   }

   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, String data, BinaryData key)
   {
      return messageDigest(algo, new character(data), key);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, Text data)
   {
      return messageDigest(algo, data, (String) null);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, Text data, String key)
   {
      return messageDigest(algo, data, new character(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, Text data, Text key)
   {
      // TODO: this implementation is "basic"; this means it does match 4GL outputs for simple
      //       tests but it has not been thoroughly tested for compatibility, especially in
      //       regard to errors and boundary conditions
      
      if (algo == null || algo.isUnknown() || data == null || data.isUnknown())
      {
         raw r = new raw();
         r.setUnknown();        
         return r;
      }
      
      return messageDigest(algorithm(algo), textToBytes(data), textToBytes(key));
   }

   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, Text data, BinaryData key)
   {
      // TODO: this implementation is "basic"; this means it does match 4GL outputs for simple
      //       tests but it has not been thoroughly tested for compatibility, especially in
      //       regard to errors and boundary conditions
      
      if (algo == null || algo.isUnknown() || data == null || data.isUnknown())
      {
         raw r = new raw();
         r.setUnknown();        
         return r;
      }
      
      return messageDigest(algorithm(algo), textToBytes(data), binaryToBytes(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, BinaryData data)
   {
      return messageDigest(algo, data, (String) null);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, BinaryData data, String key)
   {
      return messageDigest(algo, data, new character(key));
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, BinaryData data, Text key)
   {
      // TODO: this implementation is "basic"; this means it does match 4GL outputs for simple
      //       tests but it has not been thoroughly tested for compatibility, especially in
      //       regard to errors and boundary conditions
      
      if (algo == null || algo.isUnknown() || data == null || data.isUnknown())
      {
         raw r = new raw();
         r.setUnknown();        
         return r;
      }
      
      return messageDigest(algorithm(algo), binaryToBytes(data), textToBytes(key));
   }

   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    data
    *           The data to be hashed.
    * @param    key
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value.
    */
   public static raw messageDigest(character algo, BinaryData data, BinaryData key)
   {
      // TODO: this implementation is "basic"; this means it does match 4GL outputs for simple
      //       tests but it has not been thoroughly tested for compatibility, especially in
      //       regard to errors and boundary conditions
      
      if (algo == null || algo.isUnknown() || data == null || data.isUnknown())
      {
         raw r = new raw();
         r.setUnknown();
         return r;
      }
      
      return messageDigest(algorithm(algo), binaryToBytes(data), binaryToBytes(key));
   }
   
   /**
    * Converts a binary value into a hexadecimal encoded string and returns the encoded value.
    * The data will result an even number of valid hexadecimal digits.
    *
    * @param    input
    *           The text to be encoded.
    *
    * @return   The hexadecimal encoded text if the input is not unknown value, or the
    *           unknown value if that was the input. An empty binary buffer will return
    *           an empty string.
    */
   private static String hexEncodeImpl(byte[] input)
   {
      // TODO: this was written from the Progress docs and has not been tested at
      //       all; also, the boundary conditions need to be tested to confirm if this
      //       really is correct
      
      char[] res = new char[input.length * 2];
      
      for (int i = 0; i < input.length; i++)
      {
         String hex = Integer.toHexString(input[i] & 0xFF);
         boolean oneDigit = hex.length() == 1;
         res[i * 2] =  oneDigit ? '0' : hex.charAt(0);
         res[i * 2 + 1] = hex.charAt(oneDigit ? 0 : 1);
      }

      return new String(res);
   }
   
   /**
    * Hashes the specified data using the given algorithm, and returns a 16-byte 
    * binary message digest value in a {@link raw} instance.
    * 
    * @param    algo
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".
    * @param    input
    *           The data to be hashed.
    * @param    salt
    *           An optional key value to use in the hash operation.
    *           
    * @return   The hash value as a raw.  It may be the unknown value if there was an error.
    */
   private static raw messageDigest(String algo, byte[] input, byte[] salt)
   {
      byte[] hash = null;
      
      try
      {
         MessageDigest digest = MessageDigest.getInstance(algo);
               
         // TODO: do we need to check if the salt is 0 length?
         if (salt == null)
         {
            // no salt path
            hash = digest.digest(input);
         }
         else
         {
            // salt path
            digest.update(input);
            hash = digest.digest(salt);
         }
      }
      catch (Exception ex)
      {
         // TODO: improve this to match 4GL errors
         ErrorManager.recordOrThrowError(new NumberedException("Failure in hashing.", -1, ex));
      }
      
      return new raw(hash);
   }

   /**
    * Validate the message digest agorithm name and return it if it is valid.
    *
    * @param    algoName
    *           The hash algorithm to be used. This should be "MD5", "SHA-1", "SHA-256" or 
    *           "SHA-512".  <code>null</code> or unknown value should not be passed. The
    *           case does not matter.
    *
    * @return   The algorithm name as a string or <code>null</code> on any error.
    */
   private static String algorithm(character algoName)
   {
      String algo = algoName.toStringMessage();
      
      // TODO: are there any undocumented algorithm names that should be supported?
      if (!algo.equalsIgnoreCase("MD5")     &&
          !algo.equalsIgnoreCase("SHA-1")   &&
          !algo.equalsIgnoreCase("SHA-256") &&
          !algo.equalsIgnoreCase("SHA-512"))
      {
         // TODO: what error handling does the 4GL do here?
         ErrorManager.recordOrThrowError(-1, "Invalid message digest algorithm name.");
         return null;
      }
      
      return algo;
   }
   
   /**
    * Convert the given text data into a byte array that can be used for hashing. The data
    * will be converted to UTF-8 as part of the process.
    *
    * @param    data
    *           The input data to convert.
    *
    * @return   The byte array or <code>null</code> if there is no data.
    */
   private static byte[] textToBytes(Text data)
   {
      // TODO: does the 4GL have error handling for data that is 0 length or which is very large?
      if (data == null || data.isUnknown())
      {
         return null;
      }
      
      String input = data.toStringMessage();
      
      return input.getBytes(StandardCharsets.UTF_8);
   }
   
   /**
    * Convert the given binary data into a byte array that can be used for hashing.
    *
    * @param    data
    *           The input data to convert.
    *
    * @return   The byte array or <code>null</code> if there is no data.
    */
   private static byte[] binaryToBytes(BinaryData data)
   {
      // TODO: does the 4GL have error handling for data that is 0 length or which is very large?
      if (data == null || data.isUnknown())
      {
         return null;
      }
      
      return data.getByteArray();
   }
   
   /**
    * Converts memptr into a Base64 encoded text. Also checks memptr bounds.
    * <p>
    * There were reports that the error 12118 was raised (<i>Unable to allocate memory for result
    * from  function</i>) but at this moment a recreate is unavailable. It is possible there is
    * a scenario in which memory is somehow constrained on the runtime system and this error
    * would be generated.  Since there is no known recreate at this time, the corresponding
    * error processing is not implemented.
    *
    * @param   data
    *          The data to be encoded.
    *
    * @return  The encoded string. {@code unknown} will be returned if the given data is
    *          {@code null} or {@code unknown}
    */
   private static longchar base64Encode(memptr data)
   {
      if (data == null || data.isUnknown() || data.isUninitialized())
      {
         return new longchar();
      }
      
      return base64Encode(data.getByteArray());
   }
   
   /**
    * Checks if the input string format conforms to Progress rules:
    * <ol>
    *   <li>All trailing PAD symbols are ignored except the last one.</li>
    *   <li>All characters must be from the base64 alphabet. 
    *   See {@link #isBase64(char)}.
    *   </li>
    *   <li>Lets name l - length of the source string. If l % 4 == 0 than we
    *   only check the rule 2. The last char can be the PAD.
    *   </li>
    *   <li>Lets name r = l % 4. If r != 0 than we get the last r chars to 
    *   check. The r must be only 3. The last char must be the PAD. The first
    *   and second chars of the remainder part must follow rule 2. At the end
    *   if remainder is good check the main part (of the length aliquot to 4)
    *   by the rule 2.
    *   </li>
    * </ol>
    * @param   data
    *          The string to be decoded by Base64
    *          
    * @return  true if data follows Progress rules, false - doesn't follow.
    */
   private static boolean checkFormat(String data)
   {
      char[] chars = data.toCharArray();

      // ignore trailing PAD symbols except only the last one
      int len = chars.length;
      for (; len - 1 > 0; len--)
      {
         if (chars[len - 1] != PAD || chars[len - 2] != PAD)
         {
            break;
         }
      }

      int remainder = len % 4;

      // Check remainder chars first. It's faster.
      if (remainder != 0)
      {
         // common chars num in remainder must be 3 and the last is '='
         if (remainder != 3 || chars[len - 1] != PAD)
         {
            return false;
         }

         // first two symbols must be from the base64 alphabet
         if (!isBase64(chars[len - 2]) || !isBase64(chars[len - 3]))
         {
            return false;
         }
      }

      // Check all chars up to last char with index aliquot to 4
      for (int i = 0; i < len - remainder; i++)
      {
         if (!isBase64(chars[i]))
         {
            boolean lastChar = (remainder == 0 && i == len - remainder - 1);
            if (!lastChar || chars[i] != PAD)
            {
               return false;
            }
         }
      }

      // passed all checks
      return true;
   }

   /**
    * Checks if the symbol from base64 alphabet:
    * "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
    * 
    * @param   c
    *          input character to check
    * 
    * @return  true if the c is in alphabet, false - otherwise
    */
   private static boolean isBase64(char c)
   {
      return (c <= 'Z' && c >= 'A') || 
             (c <= 'z' && c >= 'a') ||
             (c <= '9' && c >= '0') ||
             (c == '+' || c == '/');
   }
   
   /**
    * Hashes an arbitrary input string into a 16-byte output string.  This conversion is one-way
    * (the original data can not be recovered) and the output is always a string of 16-bytes no
    * matter the length of the input string.
    * <p>
    * <b>WARNING: this implementation is NOT compatible with the Progress 4GL algorithm in any
    * way. Any programs relying on comparisons between data encoded by the Progress 4GL ENCODE
    * and the return value from this method will NOT compare as expected.</b>
    * <p>
    * The 16-bytes of output will only include characters in the range <code>A-Z</code> and
    * <code>a-z</code>.
    * <p>
    * This is only here because there is at least 1 customer that has installations that have
    * stored values that were encoded with the incompatible version.  Remove this at such time
    * as it is no longer in use.
    *
    * @param    data
    *           The data to be hashed. Must not be <code>null</code>.
    *
    * @return   The hashed 16-byte string.
    */
   private static String _incompatible_encode(String data)
   {
      StringBuilder sb = new StringBuilder();
      byte[] hashed = null;
      
      try
      {
         MessageDigest md = MessageDigest.getInstance("MD5");
         hashed = md.digest(data.getBytes());
         
         for (int i = 0; i < hashed.length; i++)
         {
            // this is not a fully even distribution but it is close
            // enough for an interrim solution
            int base52 = ((0x000000FF & hashed[i]) % 52); 
            sb.append(ALPHAS.charAt(base52));
         }
      }
      catch (NoSuchAlgorithmException nsa)
      {
         // we will fail below
      }
      
      if (hashed == null || hashed.length != 16)
      {
         throw new RuntimeException("Unexpected hashing result for input '" +
                                    data + "' resulting in output '" + Arrays.toString(hashed) + "'.");
      }
      
      return sb.toString();
   }
   
   /**
    * Walk forward through the array of source data provided and XOR each byte into the target
    * array, but with the index of the target matching a backwards walk.  The source data can
    * be an array of any length and the target length is not expected to match.  The target
    * indexes are calculated modulo the length of the target array, which means that for a source
    * array longer than the target array, one or more target array elements will store data XOR'd
    * from more than one source source array element (i.e. the XOR processing will wrap around).
    * For a source array smaller than the target array, there will be some low index elements
    * of the target array which will not receive any XOR'd data.  Only in the case where the
    * two arrays are the same length will there be no wrapping and no unmerged elements in the
    * target array.  Only in the case where the source array is modulo the size of the target
    * array will all elements of the target array be modified.
    * <p>
    * From a cryptographic perspective, the algorithm is quite poor.  Source arrays smaller
    * than the target arrays will leave target array bytes unmodified.  Source arrays larger
    * than the target array size will have some (or all) elements modified multiple times.
    * Both cases are causes of concern.  Not modifying elements means that there is less
    * input provided for distribution of results.  Modifying elements more than once can
    * cause issues as well.  Consider the case where the same character appears in the source
    * array in the first byte (index 0) and in the byte modulo the size of the target.  Because
    * of wrapping, this character will be XOR'd twice into the same element of the target
    * array.  XOR is its own inverse.  If you XOR the same data into a byte an even number of
    * times, then the resulting byte will be unchanged. This will have the effect of reducing
    * the distribution of the resulting data for some inputs. This is not suitable for secure
    * purposes.
    *
    * @param    source
    *           Source array to XOR from. Must not be <code>null</code>. May be of any length.
    * @param    target
    *           Target array to XOR into. Must not be <code>null</code>. May be of any length.    
    */
   private static void sourceToTargetXor(int[] source, short[] target)
   {
      int len = target.length;
      int max = len - 1;
      
      for (int i = 0; i < source.length; i++)
      {
         target[max - (i % len)] ^= source[i] & 0xFF;
      }
   }
   
   /**
    * This implements a standard CRC-16 (also known as CRC-16-IBM or CRC-16-IBM) algorithm that
    * uses a reversed polynomial of 0xA001 and swapped byte ordering. Using a reversed polynomial
    * means it processes each byte's least significant bit first (it shifts the binary data to
    * the right). Using swapped byte ordering means that the input data (which is being treated
    * as an arbitrarily large binary number) must be processed from highest element to lowest
    * element, whereas normally one might consider the most significant byte to be in the highest
    * array index position, this algorithm assumes the opposite.
    * <p>
    * If this algorithm generated uniformly distributed hashes, then in a best case scenario the
    * probability of a collision between any 2 items is (1 / 2^16) or .00152588 %.  That seems
    * good until one considers that between any 300 items there is a 50% chance of collisions
    * and between any 430 items there is a 75% chance of a collision!  This is the well known
    * birthday problem (see <a href="http://en.wikipedia.org/wiki/Birthday_attack">this page</a>).
    * <p>
    * Please note that the CRC-16 algorithm is not suitable for cryptographic hashing.  It does
    * NOT generate uniformly distributed hashes.  This means that the collision rate is not even
    * as good as the best case scenario.  Even if it did have uniform distribution, the small
    * number of bits means that collisions are not very rare.  CRC is more suitable for error
    * detection.  It should NOT be used for secure purposes.
    *
    * @param   data
    *          The bytes of data to CRC.
    * @param   crc
    *          The initial CRC value into which each element of the data will be XOR'd.
    *
    * @return  The calculated CRC value after factoring in all data bytes and binary dividing
    *          the polynomial.
    */
   private static int crc16(short[] data, int crc)
   {
      // iterate from the top of the input array to the bottom
      for (int idx = (data.length - 1); idx >= 0; idx--)
      {
         // XOR the input data into the crc
         crc ^= data[idx] & 0xFF;
         
         int bit = 7;
         
         // process each bit
         while (bit >= 0)
         {
            // check if the least significant bit is set (must be done before shifting)
            boolean lsb = ((crc & 0x01) == 1);
            
            // shift the data right by one bit
            crc >>= 1;
            
            if (lsb)
            {
               // the least significant bit was set, XOR the polynomial into the crc 
               crc ^= REVERSED_POLYNOMIAL;
            }
            
            bit--;
         }
      }
      
      return crc;
   }
   
   /**
    * Convert the source array into an array of bytes with the same number of elements, but
    * where each byte may only contain uppercase or lowercase English alphabetic characters
    * (a-z and A-Z).
    * <p>
    * The source array will have each element translated into a corresponding element in the
    * output array.  Only the least significant byte of the source array element is considered
    * in the translation algorithm.  The order of the elements in the output array will be
    * the same order as the source array (e.g. the first source element translates to the
    * first output element and so forth).
    * <p>
    * The following algorithm is used to translate the 256 possible source byte values into
    * the 52 possible output byte values:
    * <p>
    * <ol>
    *   <li> If the least significant 7 bits of the source byte are one of the 52 possible
    *        English alphabetic characters, then that is the resulting byte.  This will
    *        yield a byte with 0x41 ('A') through 0x5A ('Z') or 0x61 ('a') - 0x7A ('z'),
    *        inclusive.
    *   <li> Otherwise, the most significant (upper) nibble of the source byte will be used
    *        to select a character from 'a' through 'q'.  This nibble can only have one of
    *        16 possible values (0 through 15).  This nibble value is used as a direct
    *        index into the 16 possible lowercase output letters.  Thus, a nibble value of
    *        0 (0x0) will yield 'a' (0x61), 1 (0x1) will yield 'b' (0x62) and so on, with
    *        the last possible value of 15 (0xF) yielding 'p' (0x70).
    * </ol>
    * <p>
    * Assuming the source bytes are uniformly distributed, this translation approach is
    * guaranteed to result in a highly UNEVEN distribution of output bytes.  This is
    * cryptographically BAD and should NOT be used for secure purposes. To understand
    * why this occurs:
    * <p>
    * <pre>
    * Source Byte Range    Output Byte                               Distribution Notes
    * -----------------    -----------    ----------------------------------------------------------------------------   
    * 0x00 - 0x40          'a' - 'e'      'a' through 'd' are 16X more likely than 'e'
    * 0x41 - 0x5A          'A' - 'Z'      all equally likely (1 to 1 mapping of input and output)
    * 0x5B - 0x60          'f' - 'g'      'f' 5X more likely than 'g'
    * 0x61 - 0x7A          'a' - 'z'      all equally likely (1 to 1 mapping of input and output)
    * 0x7B - 0xC0          'h' - 'm'      'i' through 'l' are 16X more likely than 'm', 'h' is 5X more likely than 'm'
    * 0xC1 - 0xDA          'A' - 'Z'      all equally likely (1 to 1 mapping of input and output)
    * 0xDB - 0xE0          'n' - 'o'      'n' 5X more likely than 'o'
    * 0xE1 - 0xFA          'a' - 'z'      all equally likely (1 to 1 mapping of input and output)
    * 0xFB - 0xFF          'p'            only 'p' is possible
    * </pre>
    * <p>
    * More graphically, here is the exact distribution that will occur for perfectly distributed input:
    * <p>
    * <pre>
    * Character   Frequency          Histogram
    * ---------   ---------   -----------------------
    * A           2           ++
    * B           2           ++
    * C           2           ++
    * D           2           ++
    * E           2           ++
    * F           2           ++
    * G           2           ++
    * H           2           ++
    * I           2           ++
    * J           2           ++
    * K           2           ++
    * L           2           ++
    * M           2           ++
    * N           2           ++
    * O           2           ++
    * P           2           ++
    * Q           2           ++
    * R           2           ++
    * S           2           ++
    * T           2           ++
    * U           2           ++
    * V           2           ++
    * W           2           ++
    * X           2           ++
    * Y           2           ++
    * Z           2           ++
    * a           18          ++++++++++++++++++
    * b           18          ++++++++++++++++++
    * c           18          ++++++++++++++++++
    * d           18          ++++++++++++++++++
    * e           3           +++
    * f           7           +++++++
    * g           3           +++
    * h           7           +++++++
    * i           18          ++++++++++++++++++
    * j           18          ++++++++++++++++++
    * k           18          ++++++++++++++++++
    * l           18          ++++++++++++++++++
    * m           3           +++
    * n           7           +++++++
    * o           3           +++
    * p           7           +++++++
    * q           2           ++
    * r           2           ++
    * s           2           ++
    * t           2           ++
    * u           2           ++
    * v           2           ++
    * w           2           ++
    * x           2           ++
    * y           2           ++
    * z           2           ++
    * </pre>
    * <p>
    * This non-uniform distribution is purely due to the fallback approach of using the
    * upper nibble of some bytes as a selector into a subset of the possible output values.
    * Since the only a subset of the output values are targeted, it makes those values
    * more likely to occur.  In addition, because there are some upper nibble values that
    * are less likely to be encountered (because they rarely trigger the fallback mechanism
    * in the first place), this causes an additional level of non-uniformity even within
    * the subset that is possible to be selected.
    *
    * @param    source
    *           The bytes to convert.  Must not be <code>null</code>.  Only the least
    *           significant byte of each element will be used.
    *
    * @return   The converted array of bytes, with one element for each corresponding
    *           element of the source array.
    */
   private static byte[] translateToAlpha(short[] source)
   {
      byte[] target = new byte[source.length];
      
      for (int idx = 0; idx < source.length; idx++)
      {
         target[idx] = (byte)(source[idx] & 0x7F);
         
         if ((target[idx] < 'A' || (target[idx] > 'Z' && target[idx] < 'a') || target[idx] > 'z'))
         {
            target[idx] = (byte)(((source[idx] & 0xFF) >> 4) + 'a');
         }
      }
      
      return target;
   }
   
   /**
    * Encrypts or decrypts the source using the given key, initialization vector and symmetric  
    * encryption algorithm, and returns a {@link memptr} instance containing the encrypted 
    * (or decrypted) data.
    * <p>
    * This API performs validation of the specified encryption key, initialization vector and
    * encryption algorithm, following the legacy 4GL rules.
    * <p>
    * At this time, only the "AES_CBC" algorithm is supported.
    * 
    * @param    data
    *           The data to be encrypted.
    * @param    key
    *           The encryption key.
    * @param    ivValue
    *           An initialization vector to be used with the given key.
    * @param    algorithm
    *           The encryption algorithm's name.
    *                      
    * @return   See above.
    */
   private static byte[] encryptDecryptWorker(int        cipherMode, 
                                              BinaryData data,
                                              raw        key,
                                              raw        ivValue,
                                              character  algorithm)
   {
      byte[] bkey = null;
      byte[] biv = new byte[0];
      
      if (data.isUnknown())
      {
         return null;
      }
      
      if (key == null || key.isUnknown())
      {
         // TODO: defaults to SECURITY-POLICY:SYMMETRIC-ENCRYPTION-KEY; if this is unknown too, 
         // produces an error
         
         String msg = "Key must be specifed via Security-Policy:Symmetric-encryption-key " +
                      "or in " + (cipherMode == Cipher.ENCRYPT_MODE ? "ENCRYPT" : "DECRYPT");
         ErrorManager.recordOrShowError(new int[] {12121}, new String[] {msg}, 
                  false, false, false, false, false);
         return null;
      }
      else
      {
         bkey = key.getByteArray();
      }
      
      if (ivValue != null)
      {
         if (ivValue.isUnknown())
         {
            // TODO: defaults to SECURITY-POLICY:SYMMETRIC-ENCRYPTION-IV; if also unknown, then
            // do not use an initialization vector 
         }
         else
         {
            biv = ivValue.asByteArray();
         }
      }
      else
      {
         // otherwise, remains unset
      }
      

      String alg = algorithm.toStringMessage();
      Matcher matcher = KEY_LEN.matcher(alg);
      if (!matcher.matches())
      {
         // With 4GL both for ENCRYPT and DECRYPT "ENCRYPT" is used in the message.
         String msg = "Unable to determine key length in bytes, " +
                  (cipherMode == Cipher.ENCRYPT_MODE  ? "ENCRYPT" : "ENCRYPT") + " function";
         ErrorManager.recordOrShowError(new int[] {12124}, new String[] {msg}, 
                  false, false, false, false, false);
         return null;
      }
      int kl = Integer.valueOf(matcher.group(1));
      if (!(DES.matcher(alg).matches() && bkey.length * 7  == kl) && (bkey.length * 8  != kl))
      {
         String msg = "Inappropriate key length, " + 
          (cipherMode == Cipher.ENCRYPT_MODE  ? 
                "ENCRYPT" : "DECRYPT") + " function";
         ErrorManager.recordOrShowError(new int[] {12707}, new String[] {msg}, 
                                        false, false, false, false, false);
         return null;
      }

      CipherParams cipher;
      try {
         cipher = SecurityPolicyManager.getSymmetricCipherParams(alg);
      }
      catch(IllegalArgumentException e) {
         String msg = "Unsupported cipher for " +
                                 (cipherMode == Cipher.ENCRYPT_MODE  ? "ENCRYPT" : "DECRYPT");
         ErrorManager.recordOrShowError(new int[] {12146}, new String[] {msg}, 
                                 false, false, false, false, false);
         return null;
      }
      
      try
      {
         return cipher.process(data.getByteArray(), cipherMode, bkey, 
                  cipher.getIvSize() == 0 ? null : biv); 
      }
      catch (Exception e)
      {
         String msg = (cipherMode == Cipher.ENCRYPT_MODE  ? 
                "ENCRYPT" : "DECRYPT") + " error";
         ErrorManager.recordOrShowError(new int[] {14272}, new String[] {msg}, 
                  false, false, false, false, false);
         
         LOG.log(Level.SEVERE, "Error processing encrypt/decrypt", e);
      }
      
      return null;
   }
   
   /**
    * Validate the CLIENT-PRINCIPAL object, if it can be used with the specified database.
    * <p>
    * The authentication being performed depends on the CLIENT-PRINCIPAL state; currently, only
    * SSO authentication is possible - this API expects a sealed CLIENT-PRINCIPAL, and it checks
    * if the domain name and access code configuration is valid.
    * 
    * @param    clientPrincipal
    *           The CLIENT-PRINCIPAL instance.
    * @param    db
    *           The database name.
    * @param    callee
    *           where the method is called from
    *
    * @return   <code>true</code> if the object can be authenticated with the specified database.
    */
   private static boolean validateCP(ClientPrincipal clientPrincipal, String db, SealCallee callee)
   {
      if (db == null)
      {
         return false;
      }
      
      WorkArea wa = local.get();
      Persistence persistence = ConnectionManager.getPersistence(db);
      Optional<Map<QUserId, String>> users = wa.getUsers(db);
      if (!users.isPresent())
      {
         refreshUsers(wa, db, persistence);
         users = wa.getUsers(db);
      }
      refreshDomains(wa, db, persistence);
      
      Domain domain = wa.findDomain(db, clientPrincipal.getDomainName().getValue());
      // TODO: add support for special domains type and authentication callback
      boolean isDefaultDomainType = domain != null &&
                                    !SPECIAL_AUTH_SYSTEMS.contains(domain.type.name.toLowerCase());
      LoginState loginState = LoginState.valueOf(clientPrincipal.getLoginState().toStringMessage());
      
      switch (loginState)
      {
         case INITIAL:
            if ((domain != null) && (!domain.enabled || !domain.type.enabled))
            {
               ClientPrincipal.authenticationFailed(16391, db, "Domain is disabled");
               clientPrincipal.authenticationFailed("The PAM Registry domain is disabled");
               return false;
            }
            boolean ok;
            if (isDefaultDomainType)
            {
               if (!_USER_TABLE_AUTH_SYSTEM.equalsIgnoreCase(domain.type.name))
               {
                  ClientPrincipal.authenticationFailed(16392, db, 
                                                       "Domain authentication action not supported ");
                  clientPrincipal.authenticationFailed("The Domain authentication action is not allowed");
                  return false;
               }
               QUserId quid = new QUserId(clientPrincipal);
               String pwd = users.map(m -> m.get(quid)).orElse(null);
               ok = clientPrincipal.validatePassword(db, domain.accessCode, pwd, SealCallee.SET_DB_CLIENT)
                                   .booleanValue();
            }
            else
            {
               if (domain == null)
               {
                  ClientPrincipal.authenticationFailed(16388, db, "Cannot find Domain configuration ");
                  clientPrincipal.authenticationFailed("Cannot find the named PAM domain service");
                  return false;
               }
               ok = clientPrincipal.validateDomainAccessCode(db, domain.accessCode, SealCallee.SET_DB_CLIENT)
                                   .booleanValue();
            }
            
            loginState = LoginState.valueOf(clientPrincipal.getLoginState().toStringMessage());
            if (!ok && loginState == LoginState.INITIAL)
            {
               invalidState(db, clientPrincipal.sealFailure());
            }
            
            return ok;
            
         case EXPIRED:
            if (clientPrincipal.isSealed())
            {
               ClientPrincipal.validationFailed(16376, db, "User authentication expired");
               return false;
            }
            invalidState(db, clientPrincipal.sealFailure());
            return false;
            
         case LOGIN:
            if (isDefaultDomainType && !clientPrincipal.validateDomainAccessCode(domain.accessCode))
            {
               ClientPrincipal.validationFailed(16385, db, "The client-principal was corrupt");
               if (callee != SealCallee.SET_CLIENT)
               {
                  ErrorManager.recordOrShowError(new int[] { 13691 },
                           new String[] { String.format("Failed to set %s user id.", db) }, false,
                           false, false, false, false);
               }
               return false;
            }
            if ((domain != null) && (!domain.enabled || !domain.type.enabled))
            {
               ClientPrincipal.authenticationFailed(16391, db, "Domain is disabled");
               clientPrincipal.authenticationFailed("The PAM Registry domain is disabled");
               return false;
            }
            if (!isDefaultDomainType && domain == null)
            {
               if (callee == SealCallee.SET_CLIENT)
               {
                  ClientPrincipal.validationFailed(16388, db, "Cannot find Domain configuration ");
               }
               else
               {
                  ClientPrincipal.validationFailed(16389, db, "Nonexistent Domain");
               }
               return false;
            }
            return true;
            
         case LOGOUT:
            ClientPrincipal.validationFailed(16367, db, "Insufficient privileges");
            return false;
            
         case FAILED:
            ClientPrincipal.validationFailed(16369, db, "Failed account authentication");
            return false;
            
         default: // TODO: handle other states
            return false;
      }
   }

   /**
    * Report invalid CLIENT-PRINCIPAL state
    * 
    * @param where
    *        subsystem name.
    * @param sealFailure 
    *        seal failure reason
    */
   private static void invalidState(String where, SealFailure sealFailure)
   {
      String whatsUp = "is in unexpected state";
      switch (sealFailure)
      {
         case EXPIRED:
         case INVALID_STATE:
            whatsUp = "is in an invalid state";
            break;
         case MISSING_ATTRIBUTES:
            whatsUp = "was missing required attributes";
            break;
      }
      // "princpal" - as in 4GL 
      ErrorManager.recordOrShowError(new int[] {15944}, 
            new String[] {
                  "Failed " + where + " client princpal authentication because: the " +
                  "Client-Principal " + whatsUp
            },
            false, false, false, false, false);
   }

   /**
    * Validate the CLIENT-PRINCIPAL object, if it can be used with the specified database.
    * <p>
    * The authentication being performed depends on the CLIENT-PRINCIPAL state; currently, only
    * SSO authentication is possible - this API expects a sealed CLIENT-PRINCIPAL, and it checks
    * if the domain name and access code configuration is valid.
    * 
    * @param    client
    *           The CLIENT-PRINCIPAL instance.
    * @param    db
    *           The database name.
    *           
    * @return   <code>true</code> if the object can be authenticated with the specified database.
    */
   private static boolean validateClient(ClientPrincipal client, String db)
   {
      // - TODO: if unsealed, then authenticate, seal, and performed as if it was sealed 
      // - if is sealed, then validate the client-principal and set the USERID

      if (!client.isSealed())
      {
         // TODO: for now, we raise an error
         // TODO: client:PRIMARY-PASSPHRASE is required

         // TODO: if the target domain type is '_oeusertable', then use customSecurityOps
         // TODO: if the target domain type is '_oslocal', then use PAM/something else to validate??
         // TODO: if the target domain type is something else???
         return false;
      }

      if (db != null)
      {
         db = db.toLowerCase();
      }

      Domain d = local.get().findDomain(db, client.getDomainName().getValue());
      // TODO: how is client:DOMAIN-TYPE involved here?
      
      if (d == null)
      {
         // TODO: what error is it thrown?
         return false;
      }
      
      if (!d.enabled || !d.type.enabled)
      {
         // TODO: what error is it thrown?
         return false;
      }
      
      return d.accessCode.equals(client.getDomainAccessCode());
   }
   
   /**
    * Stores context-local state related to security.
    */
   public static class WorkArea
   {
      /** The default domain types. */
      private final Map<String, DomainType> defaultDomainTypes = new HashMap<>();
      
      /** The default domains. */
      private final Map<String, Domain> defaultDomains = new HashMap<>();
      
      /** The db-specific domain types. */
      private final Map<String, Map<String, DomainType>> dbDomainTypes = new HashMap<>();
      
      /** The db-specific domains. */
      private final Map<String, Map<String, Domain>> dbDomains = new HashMap<>();
      
      /** The db-specific users */
      public final Map<String, Map<QUserId, String>> dbUsers = new HashMap<>();
      
      /** The timestamps when the _sec-auth-domain were last refreshed. */
      final Map<String, Long> domainsLastRead = new HashMap<>();
      
      /** The timestamps when the _sec-auth-domain were last refreshed. */
      final Map<String, Long> domainTypesLastRead = new HashMap<>();
      
      /** The timestamps when the _user were last refreshed. */
      final Map<String, Long> usersLastRead = new HashMap<>();
      
      /**
       * Initialize the default domains and domain types.
       * <p>
       * For each directory-defined domain and domain type, you can specify either the full
       * definition, or just certain state (like disabled), if this is a default domain or domain
       * type.
       */
      public WorkArea()
      {
         // initialize the default domain types
         addDomainType(null, 
                       "_extsso", 
                       "Built-in SSO module",
                       true);
         addDomainType(null, 
                       "_oeusertable", 
                       "Built-in authentication module",
                       true);
         addDomainType(null, 
                       "_oslocal", 
                       "Built-in authentication module to the local OS user accounts",
                       true);
         
         // initialize the default domains
         addDomain(null, 
                   "", 
                   "_oeusertable", 
                   "Default user domain", 
                   null,
                   true,
                   TenantManager.DEFAULT_TENANT_NAME);
         addDomain(null,
                   "UNIX",
                   "_oslocal", 
                   "Built-in domain for UNIX user accounts, reserved for database command " +
                   "line utilities", 
                   null,
                   true,
                   TenantManager.DEFAULT_TENANT_NAME);
         addDomain(null,
                   "UNIXID",
                   "_oslocal", 
                   "Built-in SSO domain to the UNIX process user, reserved for database " +
                   "command line utilities", 
                   null,
                   true,
                   TenantManager.DEFAULT_TENANT_NAME);
         addDomain(null,
                   "WINDOWS",
                   "_oslocal", 
                   "Built-in domain for WINDOWS user accounts, reserved for database command " +
                   "line utilities", 
                   null,
                   true,
                   TenantManager.DEFAULT_TENANT_NAME);
         addDomain(null,
                   "WINDOWSID", 
                   "_oslocal", 
                   "Built-in SSO domain to the WINDOWS process user, reserved for database " + 
                   "command line utilities", 
                   null,
                   true,
                   TenantManager.DEFAULT_TENANT_NAME);
         
         // read the directory configuration
         DirectoryService ds = DirectoryService.getInstance();
         if (!ds.bind())
         {
            throw new RuntimeException("Directory bind failed");
         }

         try
         {
            String domainTypesPath = Utils.findDirectoryNodePath(ds, 
                                                                 "security-domain-types", 
                                                                 "container", 
                                                                 false);
            if (domainTypesPath != null)
            {
               String[] nodes = ds.enumerateNodes(domainTypesPath);
               
               // load domain types first
               for (String node : nodes)
               {
                  String nodeId = domainTypesPath + "/" + node;
                  Boolean enabled = ds.getNodeBoolean(nodeId, "enabled");
                  String description = ds.getNodeString(nodeId, "description");
                  String dbname = ds.getNodeString(nodeId, "dbname");
                  
                  addDomainType(dbname, node, description, enabled);
               }
            }
            
            String domainsPath = Utils.findDirectoryNodePath(ds, 
                                                             "security-domains", 
                                                             "container", 
                                                             false);
            if (domainsPath != null)
            {
               // TODO: can't enable/disable default domain with 'empty' name, as FWD doesn't allow
               // container node as 'name=""'
               
               String[] nodes = ds.enumerateNodes(domainsPath);
               
               // load domain last
               for (String node : nodes)
               {
                  String nodeId = domainsPath + "/" + node;
                  String type = ds.getNodeString(nodeId, "type");
                  Boolean enabled = ds.getNodeBoolean(nodeId, "enabled");
                  String description = ds.getNodeString(nodeId, "description");
                  String accessCode = ds.getNodeString(nodeId, "access_code");
                  String dbname = ds.getNodeString(nodeId, "dbname");
                  String tenantName = ds.getNodeString(nodeId, "tenant_name");
                  
                  addDomain(dbname, node, type, description, accessCode, enabled, tenantName);
               }
            }
         }
         finally
         {
            ds.unbind();
         }
      }
      
      /**
       * Create a domain type with the specified details.
       * 
       * @param    dbname
       *           When <code>null</code>, this will be added as a {@link #defaultDomainTypes}.
       * @param    name
       *           The domain type's name.
       * @param    description
       *           The domain type's description.
       * @param    enabled
       *           Flag indicating if this domain is enabled.
       * @return   existing or created domain type for specified database and name
       */
      public DomainType addDomainType(String dbname, String name, String description, Boolean enabled)
      {
         Map<String, DomainType> dbdt;
         if (dbname != null)
         {
            dbname = dbname.toLowerCase();
            dbdt = dbDomainTypes.computeIfAbsent(dbname, k -> new HashMap<>());
         }
         else
         {
            dbdt = defaultDomainTypes;
         }

         String dtkey = name == null ? "" : name.toLowerCase();
         DomainType domainType = findDomainType(dbname, dtkey);
         if (domainType == null)
         {
            dbdt.put(dtkey, domainType = new DomainType(name, description));
         }
         else if (dbname != null && domainType == findDomainType(null, dtkey))
         {
            // add a copy to this db-specific override, as the default was found
            dbdt.put(dtkey, domainType = new DomainType(domainType));
         }
         
         if (enabled != null)
         {
            domainType.enabled = enabled;
         }
         if (description != null)
         {
            domainType.description = description;
         }
         return domainType;
      }
      
      /**
       * Create a domain with the specified details.
       * 
       * @param    dbname
       *           When <code>null</code>, this will be added as a {@link #defaultDomains}.
       * @param    name
       *           The domain's name.
       * @param    type
       *           The domain's type.
       * @param    description
       *           The domain's description.
       * @param    accessCode
       *           The domain's access code.  If it is not specified and this is a non-default
       *           domain, then the {@link SecurityOps#DEFAULT_DOMAIN_ACCESS_CODE} value will be used.
       * @param    enabled
       *           Flag indicating if this domain is enabled.
       * @param    tenant_name
       *           The tenant name from this domain.
       */
      public void addDomain(String  dbname,
                            String  name,
                            String  type,
                            String  description,
                            String  accessCode,
                            Boolean enabled,
                            String  tenant_name)
      {
         Map<String, Domain> dbd;
         if (dbname != null)
         {
            dbname = dbname.toLowerCase();
            dbd = dbDomains.computeIfAbsent(dbname, k -> new HashMap<>());
         }
         else
         {
            dbd = defaultDomains;
         }
         
         DomainType domainType = type == null ? null : findDomainType(dbname, type);
         
         if (type == null)
         {
            throw new IllegalStateException("Domain type was not specified for domain " + name + "!");
         }
         if (domainType == null)
         {
            throw new IllegalStateException(
                  "Domain type " + type + " was not found for domain " + name + "!");
         }
         
         String dkey = name.toLowerCase();
         
         Domain domain = findDomain(dbname, dkey);
         if (domain == null)
         {
            if (accessCode == null)
            {
               accessCode = DEFAULT_DOMAIN_ACCESS_CODE;
            }
            
            dbd.put(dkey, domain = new Domain(name, domainType, description, accessCode, tenant_name));
         }
         else if (dbname != null && domain == findDomain(null, dkey))
         {
            // add a copy to this db-specific override, as the default was found
            dbd.put(dkey, domain = new Domain(domain));
         }
         
         domain.type = domainType;
         
         if (enabled != null)
         {
            domain.enabled = enabled;
         }
         if (description != null)
         {
            domain.description = description;
         }
         if (accessCode != null)
         {
            domain.accessCode = accessCode;
         }
      }
      
      /**
       * Find the domain type instance; the lookup rules are:
       * <ol>
       *    <li>if <code>dbName</code> is not specified, look in {@link #defaultDomainTypes}.</li>
       *    <li>if <code>dbName</code> is specified, look in {@link #dbDomainTypes}.</li>
       *    <li>if not found in {@link #dbDomainTypes}, look in {@link #defaultDomainTypes}.</li>
       * </ol>
       * 
       * @param    dbName
       *           The database name.  May be <code>null</code>.
       * @param    type
       *           The domain type name to look.
       *           
       * @return   The found {@link DomainType} instance, or <code>null</code> if not found.
       */
      private DomainType findDomainType(String dbName, String type)
      {
         Map<String, DomainType> dbTypes = null;
         if (dbName != null)
         {
            dbTypes = dbDomainTypes.get(dbName);
         }
         if (dbTypes == null)
         {
            dbTypes = defaultDomainTypes;
         }
         
         return dbTypes.get(type.toLowerCase());
      }
      
      /**
       * Find the domain instance; the lookup rules are:
       * <ol>
       *    <li>if <code>dbName</code> is not specified, look in {@link #defaultDomains}.</li>
       *    <li>if <code>dbName</code> is specified, look in {@link #dbDomains}.</li>
       *    <li>if not found in {@link #dbDomains}, look in {@link #defaultDomains}.</li>
       * </ol>
       * 
       * @param    dbName
       *           The database name.  May be <code>null</code>.
       * @param    domain
       *           The domain name to look.
       *           
       * @return   The found {@link Domain} instance, or <code>null</code> if not found.
       */
      private Domain findDomain(String dbName, String domain)
      {
         Map<String, Domain> dbDomains = null;
         if (dbName != null)
         {
            dbDomains = this.dbDomains.get(dbName.toLowerCase());
         }
         if (dbDomains == null)
         {
            dbDomains = defaultDomains;
         }
         
         return dbDomains.get(domain.toLowerCase());
      }
      
      /**
       * Add database user.
       * 
       * @param   dbName
       *          The database name.
       * @param   quid
       *          The qualified user id.
       * @param   password
       *          The user's password.
       */
      public void addUser(String dbName, QUserId quid, String password)
      {
         Map<QUserId, String> users = this.dbUsers.computeIfAbsent(dbName, k -> new HashMap<>());
         users.put(quid, password);
      }
      
      /**
       * Get map of database user's passwords by qualified user id
       * 
       * @param   dbName
       *          The database name
       *
       * @return map of database user's passwords by qualified user id
       */
      public Optional<Map<QUserId, String>> getUsers(String dbName)
      {
         Map<QUserId, String> users = this.dbUsers.get(dbName);
         return users == null ? Optional.empty() : 
            Optional.of(Collections.unmodifiableMap(users));
      }
   }
   
   /**
    * Find the domain type instance; the lookup rules are:
    * <ol>
    *    <li>if <code>dbName</code> is not specified, look in defaultDomainTypes.</li>
    *    <li>if <code>dbName</code> is specified, look in dbDomainTypes.</li>
    *    <li>if not found in dbDomainTypes, look in defaultDomainTypes.</li>
    * </ol>
    *
    * @param    dbName
    *           The database name.  May be <code>null</code>.
    * @param    type
    *           The domain type name to look.
    *
    * @return   The found {@link DomainType} instance, or <code>null</code> if not found.
    */
   public static DomainType findDomainType(String dbName, String type)
   {
      return local.get().findDomainType(dbName, type);
   }
   
   /**
    * Create a domain type with the specified details.
    *
    * @param    dbname
    *           When <code>null</code>, this will be added as a defaultDomainTypes.
    * @param    name
    *           The domain type's name.
    * @param    description
    *           The domain type's description.
    * @param    enabled
    *           Flag indicating if this domain is enabled.
    *
    * @return   existing or created domain type for specified database and name
    */
   public static DomainType addDomainType(String dbname, String name, String description, Boolean enabled)
   {
      return local.get().addDomainType(dbname, name, description, enabled);
   }

   /**
    * Find the domain instance; the lookup rules are:
    * <ol>
    *    <li>if <code>dbName</code> is not specified, look in defaultDomains.</li>
    *    <li>if <code>dbName</code> is specified, look in dbDomains.</li>
    *    <li>if not found in dbDomains, look in defaultDomains.</li>
    * </ol>
    * 
    * @param    dbName
    *           The database name.  May be <code>null</code>.
    * @param    domain
    *           The domain name to look.
    *
    * @return   The found {@link Domain} instance, or <code>null</code> if not found.
    */
   public static Domain findDomain(String dbName, String domain)
   {
      return local.get().findDomain(dbName, domain);
   }

   /**
    * Get map of database user's passwords by qualified user id
    * 
    * @param dbName
    *        the database name
    *
    * @return map of database user's passwords by qualified user id
    */
   public static Optional<Map<QUserId, String>> getUsers(String dbName)
   {
      return local.get().getUsers(dbName); 
   }
   
   /**
    * Remove previously loaded information about domain types, domains, and users for a database and then
    * reload all over again making sure their up-to-date version id stored in memory.
    *
    * @param   dbName
    *          database name
    * @param   persistence
    *          The {@code Persistence} for the database.
    */
   static void refreshUsersAndDomains(String dbName, Persistence persistence)
   {
      WorkArea wa = local.get();
      refreshUsers(wa, dbName, persistence);
      refreshDomains(wa, dbName, persistence);
   }
   
   /** Qualified user id. */
   public static class QUserId
   {
      /** The user's name, in upper-case. */
      private final String uid;
      
      /** The user's domain, also in upper-case. */
      private final String domain;
      
      /** Precomputed hash code for this object. */
      private final int hc;
      
      /**
       * Constructor.
       * 
       * @param uid
       *        user id
       * @param domain
       *        domain name 
       */
      public QUserId(String uid, String domain)
      {
         this.uid = uid == null ? "" : uid.toUpperCase();
         this.domain = domain == null ? "" : domain.toUpperCase();
         
         // pre-compute the hash code 
         int prime = 31;
         int result = 1;
         result = prime * result + ((domain == null) ? 0 : domain.hashCode());
         hc = prime * result + ((uid == null) ? 0 : uid.hashCode());
      }
      
      /**
       * Constructor.
       * 
       * @param cp
       *        {@link ClientPrincipal}
       */
      public QUserId(ClientPrincipalResource cp)
      {
         this(cp.getUserId().value, cp.getDomainName().value);
      }

      /**
       * Returns a hash code value for the object.
       * 
       * @return a hash code value for this object.
       */
      @Override
      public int hashCode()
      {
         return hc;
      }
      
      /**
       * Indicates whether some other object is "equal to" this one.
       * 
       * @param   obj   the reference object with which to compare.
       * @return  {@code true} if this object is the same as the obj
       *          argument; {@code false} otherwise.
       */
      @Override
      public boolean equals(Object obj)
      {
         if (this == obj)
         {
            return true;
         }
         if (obj == null)
         {
            return false;
         }
         if (getClass() != obj.getClass())
         {
            return false;
         }
         
         QUserId other = (QUserId) obj;
         if (domain == null)
         {
            if (other.domain != null)
            {
               return false;
            }
         }
         else if (!domain.equals(other.domain))
         {
            return false;
         }
         
         if (uid == null)
         {
            if (other.uid != null)
            {
               return false;
            }
         }
         else if (!uid.equals(other.uid))
         {
            return false;
         }
         
         return true;
      }
   }
   
   /**
    * This defines the configuration of a security domain, with state taken from the 4GL legacy 
    * <code>_sec-authentication-system.d</code> system table.
    * <p>
    * When DomainType is enabled then <code>false</code>, all domains having this type will
    * be deactivated.
    */
   public static class DomainType
   {
      /** The domain type's name. */
      private final String name;
      
      /** Flag indicating if this domain type is enabled. */
      private boolean enabled;
      
      /** The domain type's description. */
      private String description;
      
      /**
       * Create a new domain type and initialize it with the given state.
       * 
       * @param    name
       *           The domain's name.
       * @param    description
       *           The domain's description.
       */
      public DomainType(String name, String description)
      {
         this(name, description, true);
      }
      
      /**
       * Create a new domain type and initialize it with the given state.
       * 
       * @param    name
       *           The domain's name.
       * @param    description
       *           The domain's description.
       * @param    enabled
       *           Flag indicating if this domain type is enabled.
       */
      public DomainType(String name, String description, boolean enabled)
      {
         this.name = name;
         this.description = description;
         this.enabled = enabled;
      }

      /**
       * Create a new domain type and copy the state from the specified source.
       * 
       * @param    src
       *           The source domain type.
       */
      public DomainType(DomainType src)
      {
         this.name = src.name;
         this.enabled = src.enabled;
         this.description = src.description;
      }

      /**
       * Get domain type name.
       * @return domain type name.
       */
      public String getName()
      {
         return name;
      }

      /**
       * Check if domain type is enabled.
       * @return <code>true</code> if domain type is enabled.
       */
      public boolean isEnabled()
      {
         return enabled;
      }

      /**
       * Get domain type description.
       * @return domain type description.
       */
      public String getDescription()
      {
         return description;
      }
   }
   
   /**
    * This defines the configuration of a security domain, with state taken from the 4GL legacy 
    * <code>_sec-authentication-domain.d</code> system table.
    * <p>
    * The  Domain accessCode is the unencrypted value of the access code.
    */
   public static class Domain
   {
      /** The domain's name. */
      private final String name;
      
      /** The domain's type. */
      private DomainType type;
      
      /** Flag indicating if this domain is enabled. */
      private boolean enabled;
      
      /** The domain's description. */
      private String description;
      
      /** The domain's access code. */
      private String accessCode;
      
      /** The tenant name for this domain. */
      private String tenantName;
      
      /**
       * Create a new domain and initialize it with the given state.
       * 
       * @param   name
       *          The domain's name.
       * @param   type
       *          The domain's type.
       * @param   description
       *          The domain's description.
       * @param   accessCode
       *          The domain's access code.
       * @param   tenantName
       *          The tenant name for this domain.
       */
      public Domain(String name, DomainType type, String description, String accessCode, String tenantName)
      {
         this(name, type, true, description, accessCode, tenantName);
      }
      
      /**
       * Create a new domain and initialize it with the given state.
       * 
       * @param   name
       *          The domain's name.
       * @param   type
       *          The domain's type.
       * @param   enabled
       *          Flag indicating if this domain is enabled.
       * @param   description
       *          The domain's description.
       * @param   accessCode
       *          The domain's access code.
       * @param   tenantName
       *          The tenant name for this domain.
       */
      public Domain(String     name, 
                    DomainType type,
                    boolean    enabled,
                    String     description,
                    String     accessCode,
                    String     tenantName)
      {
         this.name = name;
         this.type = type;
         this.description = description;
         this.accessCode = accessCode;
         this.enabled = enabled;
         this.tenantName = (tenantName == null) ? TenantManager.DEFAULT_TENANT_NAME : tenantName;
      }
      
      /**
       * Create a new domain and copy the state from the specified source.
       * 
       * @param    src
       *           The source domain.
       */
      public Domain(Domain src)
      {
         this.name = src.name;
         this.type = src.type;
         this.enabled = src.enabled;
         this.description = src.description;
         this.accessCode = src.accessCode;
         this.tenantName = src.tenantName;
      }
      
      /**
       * Get the domain name.
       * 
       * @return  The domain name.
       */
      public String getName()
      {
         return name;
      }
      
      /**
       * Get the domain type.
       * 
       * @return  The domain type.
       */
      public DomainType getType()
      {
         return type;
      }
      
      /**
       * Checks if this domain is enabled. 
       * 
       * @return  {@code true} if this domain is enabled.
       */
      public boolean isEnabled()
      {
         return enabled;
      }
      
      /**
       * Get domain description.
       * 
       * @return  The domain description.
       */
      public String getDescription()
      {
         return description;
      }
      
      /**
       * Get domain access code.
       * 
       * @return  The domain access code.
       */
      public String getAccessCode()
      {
         return accessCode;
      }
      
      /**
       * Get the tenant name for this domain.
       *
       * @return  the tenant name for this domain.
       */
      public String getTenantName()
      {
         return tenantName;
      }
   }
}