NativeInvoker.java

/*
** Module   : NativeInvoker.java
** Abstract : support for invocation of shared library functions 
**
** Copyright (c) 2013-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- ----------------------------------------Description---------------------------------------
** 001 GES 20140116 First version.
** 002 GES 20140127 Fixed memptr processing and pending error detection logic in
**                  copyDataOutWorker().
** 003 GES 20140212 Fixed the copy back of return data and the processing for output arrays.
**                  Lots of tweaks to error processing (eliminating prefixes from error messages
**                  and ensuring that unsuppressed errors get generated at the right times).
**                  Fixed output parameter processing for integers set to unknown value. Added
**                  processing to fixup input parameters that are not BDT, but are Java wrappers
**                  (like Integer). Fixed integer copy in processing for scalar instances. 
** 004 HC  20140613 Improved extent parameter support - introduced multiple wrapper classes 
**                  depending on the parameter type and the l-value type (variable vs.
**                  field).
** 005 ECF 20150715 Replace StringBuffer with StringBuilder.
** 006 GES 20171220 Changes to match new silent mode implementation.
** 007 CA  20181029 Some changes to allow native API calls via the CALL handle's DLL-CALL-TYPE
**                  mode. 
** 008 CA  20210609 Reworked INPUT/INPUT-OUTPUT parameters to a new approach, where they are explicitly
**                  initialized at the method's execution, and not at the caller's arguments.
**     CA  20220515 Allow library and memptr calls to be executed on server-side.
**     CA  20220602 A memptr returned by a native library call will not have its size set.  Removed the length
**                  validation for memptr arguments.
** 009 SVL 20230803 genInputUnknownErr was made public.
** 010 GBB 20240826 OSResourceManager moved to a new package osresource.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.library;

import java.util.*;
import java.math.*;

import com.goldencode.p2j.util.*;
import com.goldencode.p2j.util.osresource.*;
import com.goldencode.p2j.security.*;
import com.goldencode.util.*;

/**
 * Server-side support for native library management and function invocation.  As much as is
 * possible, parameter validation, signature normalization and error processing will occur
 * in the class.  All library loading/unloading, function address lookup, marshalling/
 * unmarshalling of parameters and the actual invocation of the function will be done on the
 * client-side (see {@link LibraryDaemon} and {@link LibraryManager}).
 * <p>
 * All the functionality of this class (actually, of the entire package) is exposed via two
 * methods, {@link #invoke} and {@link #release}.
 */
public class NativeInvoker
{
   /** Stores context local instances of the remote memory manager proxy. */
   private static final ContextLocal<WorkArea> ctxt = new ContextLocal<WorkArea>()
   {
      protected WorkArea initialValue()
      {
         WorkArea wa = new WorkArea();
         
         wa.libmgr = OSResourceManager.getLibrary();
         
         return wa;
      }
   };
   
   /**
    * Instances of this class cannot be constructed.
    */
   private NativeInvoker()
   {
   }
   
   /**
    * Invoke the defined native API call in a manner that is compatible with how the 4GL behaves.
    * 
    * @param    libname
    *           The library name where this native procedure should be found.
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    * @param    signInput
    *           The signature represented by the caller's passed arguments. This is NOT the same
    *           as the actual signature of native API to be called. This will be compared to the
    *           native API's required signature and errors will be raised as needed.
    * @param    modes
    *           An encoded string with the mode of each RUN parameter specified as a character
    *           in the string.  This is what was defined on the RUN statement NOT what was
    *           defined in the PROCEDURE EXTERNAL.
    * @param    argsInput
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.
    */
   public static void invoke(String     libname,
                             String     pname,
                             String     iename,
                             Class<?>[] signInput,
                             String     modes,
                             Object...  argsInput)
   {
      InternalEntry ie = SourceNameMapper.getInternalEntry(pname, iename, false);
      
      if (ie == null)
      {
         // this is some kind of conversion or configuration problem, since we should never be
         // called unless there is a matching definition
         String err1 = "Missing InternalEntry for library " + libname + " and external proc " +
                       pname + " and internal entry name " + iename;
         throw new IllegalStateException(err1);
      }
      
      NativeAPIEntry calldef = null;
      
      if (ie instanceof NativeAPIEntry)
      {
         calldef = (NativeAPIEntry) ie;
      }
      
      if (calldef == null)
      {
         // this is some kind of conversion or configuration problem, since we should never be
         // called unless there is a matching definition
         String err2 = "Missing NativeAPIEntry for library " + libname + " and external proc " +
                       pname + " and internal entry name " + iename;
         throw new IllegalStateException(err2);
      }
      
      invoke(calldef, false, pname, signInput, modes, argsInput);
   }
   
   /**
    * Invoke the defined native API call in a manner that is compatible with how the 4GL behaves.
    * 
    * @param    calldef
    *           The native API definition.
    * @param    noInvoke
    *           Flag indicating the native API will not be invoked, just resolved (and its library
    *           loaded).
    * @param    signInput
    *           The signature represented by the caller's passed arguments. This is NOT the same
    *           as the actual signature of native API to be called. This will be compared to the
    *           native API's required signature and errors will be raised as needed.
    * @param    modes
    *           An encoded string with the mode of each RUN parameter specified as a character
    *           in the string.  This is what was defined on the RUN statement NOT what was
    *           defined in the PROCEDURE EXTERNAL.
    * @param    argsInput
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.
    */
   public static void invoke(NativeAPIEntry calldef,
                             boolean        noInvoke,
                             String         pname,
                             Class<?>[]     signInput,
                             String         modes,
                             Object...      argsInput)
   {
      String libname = calldef.getLibraryName();
      String iename = calldef.getLegacyName();
      
      // at this point, calldef should be an existing native API definition
      boolean           persistent = calldef.isPersistent();
      CallingConvention callconv   = calldef.getCallingConvention();
      int               ordinal    = calldef.getOrdinal();
      List<Parameter>   parms      = calldef.getParameterList();
      String            legacy     = calldef.getLegacyName();
      Signature         briefcase  = new Signature();
      Class<?>[]        signature  = new Class<?>[signInput.length];
      Object[]          args       = new Object[argsInput.length];
      
      // before we can properly process the arguments, we must eliminate any use of the
      // OutputExtentParameter container class
      fixupArrays(signInput, signature, argsInput, args);
      
      // input parameters can be passed as literals in the 4GL, some of which will emit
      // in Java as literals instead of BDT wrappers; with autoboxing the resulting instances
      // may include String, Byte, Short, Integer, Long, Float and Double; these will be
      // translated to the proper BDT wrapper so that downstream processing is simpler
      fixupNonWrappers(signature, args);
      
      boolean fail = true;
      briefcase.setNoInvoke(noInvoke);
      
      // do inbound error checking, prepare the return value/parameter descriptors and copy
      // the input data into the descriptors
      if (inboundProcessing(pname, iename, parms, signature, modes, briefcase, args))
      {
         fail = false;
         return;
      }
      
      NativeAPICaller libmgr = ctxt.get().libmgr;
      
      // make the call to the client-side (we use the "legacy" name from the original def
      // rather than the RUN statement iename, since the iename can have non-matching case
      // but the legacy name must have matched the library's exported name exactly)
      briefcase = libmgr.invoke(libname, legacy, ordinal, persistent, callconv, briefcase);
      
      int rc = briefcase.getErrorCode();
      
      // error processing
      if (rc != LibraryManager.ERROR_NONE)
      {
         switch (rc)
         {
            case LibraryManager.ERROR_LIBRARY_LOAD:
               genLibraryLoadErr(libname, briefcase.getPlatform());
               break;
            case LibraryManager.ERROR_MISSING_ENTRYPOINT:
               genEntrypointErr(iename, ordinal);
               break;
            // none of these cases should ever occur, so we throw a "real" java exception
            case LibraryManager.ERROR_INVALID_CALL_CONV:
            case LibraryManager.ERROR_INVALID_TYPE:
            case LibraryManager.ERROR_UNKNOWN_FAILURE:
            case LibraryManager.ERROR_OUT_OF_MEMORY:
               String msg = String.format("Unexpected error in JNI dispatch(), rc = %d", rc);
               throw new IllegalArgumentException(msg);
         }
         
         fail = false;
         return;
      }
      
      // outbound checks and copy data back
      outboundProcessing(parms, signature, briefcase, args);
      fail = false;
   }
   
   /**
    * Attempts to unload the specified library from memory (this is the implementation of the
    * RELEASE EXTERNAL statement).  This code delegates processing the to client-side.
    * 
    * @param    libname
    *           The name of the library to attempt to unload.
    */
   public static void release(String libname)
   {
      ctxt.get().libmgr.release(libname);
   }

   /**
    * Generate an input value cannot be unknown error.
    *
    * @param    force
    *           <code>true</code> to throw the error even in silent mode.
    */
   public static void genInputUnknownErr(boolean force)
   {
      String spec = "A variable or array element passed as an INPUT or INPUT-OUTPUT parameter " +
                    "to a DLL cannot contain the Unknown value";

      genForcableError(12272, spec, force);
   }
   
   /**
    * Eliminate any use of the OutputExtentParameter container class and replace it with the
    * actual array references.  The passed references will be edited if necessary.
    * 
    * @param    signIn
    *           The signature represented by the caller's passed arguments.
    * @param    signOut
    *           The fixed signature.
    * @param    argsIn
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.
    * @param    argsOut
    *           The fixed arguments.
    */
   private static void fixupArrays(Class<?>[] signIn,
                                   Class<?>[] signOut,
                                   Object[]   argsIn,
                                   Object[]   argsOut)
   {
      for (int i = 0; i < argsIn.length; i++)
      {
         if (argsIn[i] instanceof AbstractExtentParameter)
         {
            AbstractExtentParameter oep = (AbstractExtentParameter) argsIn[i];
            
            argsOut[i] = oep.getVariableSafe();
            signOut[i] = argsOut[i].getClass();
            
            // disable the logic that tries to change the reference since we will never
            // have an indeterminate sized extent
            oep.setParameter(oep.getVariableSafe());
         }
         else
         {
            argsOut[i] = argsIn[i];
            signOut[i] = signIn[i];
         }
      }
   }
   
   /**
    * Fixup any input parameters which were passed as literals in the 4GL, which emitted
    * as Java literals instead of BDT wrappers.  With autoboxing the resulting instances
    * may include String, Byte, Short, Integer, Long, Float and Double.  These will be
    * translated to the proper BDT wrapper so that downstream processing is simpler.
    * <p>
    * At this time there is no actual checking of whether this is really an input mode parm
    * or what the exact type should be. If there is a mismatch on the mode, then the caller
    * has coded something that can never receive an update and they will have to figure that
    * out themselves (but it should never happen in converted code since the 4GL disallows
    * literals except for input parms). If there is a type mismatch, then it will trigger a
    * failure downstream in a manner that is 4GL compatible.
    *
    * @param    signature
    *           The signature represented by the caller's passed arguments.  This will be edited
    *           directly to fixup any issues.
    * @param    args
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.   This will be edited directly to fixup any issues.
    */
   private static void fixupNonWrappers(Class<?>[] signature, Object[] args)
   {
      for (int i = 0; i < signature.length; i++)
      {
         if (signature[i].equals(Byte.class)    ||
             signature[i].equals(Short.class)   ||
             signature[i].equals(Integer.class) ||
             signature[i].equals(Long.class))
         {
            signature[i] = int64.class;
            args[i]      = new int64((Number) args[i]);
         }
         else if (signature[i].equals(Float.class) || signature[i].equals(Double.class))
         {
            signature[i] = decimal.class;
            args[i]      = new decimal((Number) args[i]);
         }
         else if (signature[i].equals(String.class))
         {
            signature[i] = character.class;
            args[i]      = new character((String) args[i]);
         }
      }
   }
   
   /**
    * Checks the passed parameters (and their values) against the definition and raises errors
    * to match the behavior of the 4GL, if OK the data will be copied into the descriptors for
    * transport.
    * <p>
    * The following checks are done (in order):
    * <ol>
    *    <li> unknown value, uninitialized pointer and empty string tests
    *    <li> type and mode of the parameters
    *    <li> value of the parameters
    *    <li> number of parameters
    * </ol>
    * <p>
    * This code (and its downstream workers) also handles the proper setup of the descriptors
    * for the return value and the parameters.  Those descriptors define whether the particular
    * parameter needs to be passed as a pointer, whether it copies data back and other metadata
    * as well as containing the actual value(s) to be transported to the client.
    * <p>
    * The complexity and quirkiness of the error checking naturally makes this the proper place
    * to setup the descriptors (including copying in the input data).
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    * @param    defs
    *           Parameter definitions. This defines the required signature of the actual native
    *           API as defined by a converted 4GL <code>PROCEDURE EXTERNAL</code> statement. As
    *           such it may or may not match reality, but it is what the 4GL uses for error
    *           checking (and there isn't anything better that we can do).
    * @param    signature
    *           The class list represented by the caller's passed arguments (determined at 
    *           runtime). This is NOT the same as the actual signature of native API to be
    *           called. This will be compared to the native API's required signature and errors
    *           will be raised as needed.
    * @param    modetxt
    *           An encoded string with the mode of each RUN parameter specified as a character
    *           in the string.  This is what was defined on the RUN statement NOT what was
    *           defined in the PROCEDURE EXTERNAL.
    * @param    briefcase
    *           The set of return value and parameter descriptors which should be setup for
    *           transport to the client.  This is initially passed as an empty instance and
    *           if no errors are encountered, it will be fully configured for transport to the
    *           client on return.
    * @param    args
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.
    *
    * @return   <code>true</code> to cause a silent return from the calling method. Of course, if
    *           and error has been raised, then the exception will be unwinding the stack and the
    *           return won't matter. But in the case where silent error mode is enabled (the 4GL
    *           <code>NO-ERROR</code> case), we must return back silently.
    */
   private static boolean inboundProcessing(String          pname,
                                            String          iename,
                                            List<Parameter> defs,
                                            Class<?>[]      signature,
                                            String          modetxt,
                                            Signature       briefcase,
                                            Object...       args)
   {
      int numDefs = defs.size();
      
      // initialize the argument list
      initializeBriefcase(defs, signature, briefcase, args);
      
      // we only error check the smaller of the number of defined parms or passed parms
      int numCheck = Math.min(numDefs, signature.length);
      
      int offset = 0;
      
      // check 1: unknown value and empty string tests
      for (int i = 0; i < numCheck; i++)
      {
         Parameter      parm = defs.get(i);
         String         mode = parm.getMode();
         BaseNativeType bnt  = null;
         
         if ("RETURN".equalsIgnoreCase(mode))
         {
            // compensate for the missing return parameter when looking up the BNT
            offset++;
            bnt = briefcase.getReturnValue();
         }
         else
         {
            bnt = briefcase.getArgument(i - offset);
         }
         
         // avoid NPE in checkUnknown, we will fail later in inbound processing during the
         // type checking
         if (bnt == null)
            continue;
         
         // this also will edit the descriptor(s) if needed to disable copying back as matches
         // the 4GL behavior (it is much better to do it here than to duplicate this logic
         // elsewhere)
         if (checkUnknown(pname, iename, parm.getMode(), signature[i], args[i], bnt))
            return true;
      }
      
      // make sure our encoded mode text is regular
      if (modetxt == null)
      {                                     
         modetxt = StringHelper.repeatChar(InternalEntry.INPUT_MODE, numDefs);
      }
      else
      {
         int gap = numDefs - modetxt.length(); 
         
         if (gap > 0)
         {
            StringBuilder sb = new StringBuilder(modetxt);
            StringHelper.repeatChar(InternalEntry.INPUT_MODE, gap, sb);
            modetxt = sb.toString();
         }
      }
      
      // check 2: type and mode of the parameters
      for (int j = 0; j < numCheck; j++)
      {
         Parameter parm = defs.get(j);
         Class<?> real = signature[j].isArray() ? signature[j].getComponentType() : signature[j];
         
         if (checkModeAndType(pname, iename, parm, modetxt.charAt(j), real))
            return true;
      }
      
      // reset our offset processing
      offset = 0;
      
      // check 3: value of the parameters (only input and input-output integer data is checked,
      // as well as uninitialized memptr and some array errors)
      for (int k = 0; k < numCheck; k++)
      {
         Parameter parm = defs.get(k);
         String    type = parm.getType();
         String    mode = parm.getMode();
         
         // detect use of an indeterminate array
         if (signature[k].isArray() && ArrayAssigner.isDynamicArray((BaseDataType[]) args[k]))
         {
            genIndeterminateArrayErr();
            return true;
         }
         
         if ("RETURN".equalsIgnoreCase(mode))
         {
            if (signature[k].isArray())
            {
               genNoArrayForReturnErr();
               return true;
            }
            
            // compensate for the missing return parameter when looking up the BNT
            offset++;
            continue;
         }
         
         BaseNativeType bnt = briefcase.getArgument(k - offset);
         
         if ("INPUT".equals(mode) || "INPUT-OUTPUT".equals(mode))
         {
            Class<?> real = signature[k].isArray() ? signature[k].getComponentType()
                                                   : signature[k];
                                                   
            if (real.equals(integer.class) || real.equals(int64.class))
            {
               if (checkIntegerRange(type, signature[k], args[k]))
                  return true;
            }
            else if (signature[k].equals(memptr.class))
            {
               // the above test is deliberately done against the original class instead of
               // the "real" class because this error is only raised when we are dealing with
               // the scalar (non-array) case; that is why it is safe to cast this directly
               // to a memptr when normally we could have either a memptr or memptr[]
               memptr ptr = (memptr) args[k];
               
               if (ptr.isUninitialized()) //  || ptr.lengthOf() == 0)
               {
                  genUninitMemptrErr(pname, iename);
                  return true;
               }
            }
            
            // now we have passed our input checks, let's copy our data in for this parm
            copyDataIn(type, mode, signature[k], args[k], bnt);
         }
         
         // pass-by-pointer cases are special because they actually copy in as well; if
         // we don't copy our data in then the native call will trap badly because there
         // won't be any buffer to access
         if ("OUTPUT".equals(mode) && bnt.isPassByPointer())
         {
            copyDataIn(type, mode, signature[k], args[k], bnt);
         }
      }
      
      // check 4: mismatching number of parameters (this could have been done first, but in the
      // 4gl it is last), this includes the return parm if it exists
      if (numDefs != signature.length)
      {
         String spec = "Mismatched number of parameters passed to routine %s %s";
         ErrorManager.recordOrThrowError(3234, String.format(spec, iename, pname), false);
         return true;
      }
      
      return false;
   }
   
   /**
    * Analyze the required parameter signature for the function and the passed arguments and
    * use that information to setup the initial configuration of the briefcase.
    *
    * @param    defs
    *           Parameter definitions. This defines the required signature of the actual native
    *           API as defined by a converted 4GL <code>PROCEDURE EXTERNAL</code> statement.
    * @param    signature
    *           The class list represented by the caller's passed arguments (determined at 
    *           runtime). This is NOT the same as the actual signature of native API to be
    *           called. This will be compared to the native API's required signature and errors
    *           will be raised as needed.
    * @param    briefcase
    *           The set of return value and parameter descriptors which should be setup for
    *           transport to the client.  This is initially passed as an empty instance.
    * @param    args
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.
    */
   private static void initializeBriefcase(List<Parameter> defs,
                                           Class<?>[]      signature,
                                           Signature       briefcase,
                                           Object...       args)
   {
      int numDefs = defs.size();
      int offset  = 0;
      
      briefcase.setArgumentSize(calcNumArguments(defs));
      
      // we can only process the smaller of these values, the code will fail downstream
      // with a mismatching number of parms
      int num = Math.min(numDefs, signature.length);
      
      for (int j = 0; j < num; j++)
      {
         Parameter parm = defs.get(j);
         
         String  type     = parm.getType();
         String  mode     = parm.getMode();
         String  attr     = parm.getAttribute("handle_to");
         boolean handleTo = "true".equalsIgnoreCase(attr);
         
         BaseNativeType bnt = null;
         
         if (signature[j].isArray())
         {
            BaseDataType[]   bdtArray = (BaseDataType[]) args[j];
            BaseNativeType[] bntArray = new BaseNativeType[bdtArray.length];
            
            NativeTypeArray nta = new NativeTypeArray();
            
            boolean copyIn  = false;
            boolean copyOut = false;
            
            if ("INPUT".equalsIgnoreCase(mode) || "INPUT-OUTPUT".equalsIgnoreCase(mode))
            {
               copyIn = true;
            }
            
            if ("OUTPUT".equalsIgnoreCase(mode) || "INPUT-OUTPUT".equalsIgnoreCase(mode))
            {
               copyOut = true;
            }
            
            // no RETURN processing is needed, return mode can't be EXTENT
            
            nta.setCopyIn(copyIn);
            nta.setCopyOut(copyOut);
            nta.setPassByPointer(true);
            
            // initialize the array of elements
            for (int k = 0; k < bntArray.length; k++)
            {
               bntArray[k] = createScalarInstance(type, mode, handleTo);
               
               if (bntArray[k] instanceof NativeInteger)
               {
                  NativeInteger ni = (NativeInteger) bntArray[k];
                  ni.setInArray(true);
               }
            }
            
            // save the elements off
            nta.setElements(bntArray);
            bnt = nta;
         }
         else
         {
            bnt = createScalarInstance(type, mode, handleTo);
         }
         
         if ("RETURN".equalsIgnoreCase(mode))
         {
            briefcase.setReturnValue(bnt);
            offset++;
         }
         else
         {
            // we use an index offset to compensate for a return value that is arbitrary
            // inserted into the parameter list (instead of keeping it separate, the 4GL
            // allows the programmer to define it in an arbitrary position)
            briefcase.setArgument(bnt, (j - offset));
         }
      }
   }
   
   /**
    * Return back the non-negative number of arguments for this native API call by processing
    * the parameter definitions and excluding any RETURN parameter that is found.
    *
    * @param    defs
    *           Parameter definitions. This defines the required signature of the actual native
    *           API as defined by a converted 4GL <code>PROCEDURE EXTERNAL</code> statement.
    *
    * @return   The non-negative number of non-RETURN arguments.
    */
   private static int calcNumArguments(List<Parameter> defs)
   {
      if (defs == null)
      {
         return 0;
      }
      
      int max = defs.size();
      
      for (int i = 0; i < max; i++)
      {
         Parameter parm = defs.get(i);
         
         if ("RETURN".equalsIgnoreCase(parm.getMode()))
         {
            max--;
            break;
         }
      }
      
      return max;
   }
   
   /**
    * Instantiate a placeholder for a single scalar parameter and configure it based on the
    * type and mode.
    *
    * @param    type
    *           The 4GL "DLL" data type.
    * @param    mode
    *           The 4GL parameter mode (e.g. INPUT).
    * @param    handleTo
    *           <code>true</code> if HANDLE TO was specified.
    *
    * @return   The new placeholder instance.
    */
   private static BaseNativeType createScalarInstance(String type, String mode, boolean handleTo)
   {
      BaseNativeType bnt = null;
      
      boolean copyIn  = false;
      boolean copyOut = false;
      boolean asPtr   = false;
      
      if ("INPUT".equalsIgnoreCase(mode))
      {
         copyIn = true;
         asPtr  = handleTo;
      }
      
      if ("INPUT-OUTPUT".equalsIgnoreCase(mode))
      {
         copyIn = true;
      }
      
      if ("OUTPUT".equalsIgnoreCase(mode) || "INPUT-OUTPUT".equalsIgnoreCase(mode) ||
          "RETURN".equalsIgnoreCase(mode))
      {
         copyOut = true;
      }
      
      if ("CHARACTER".equalsIgnoreCase(type) || "LONGCHAR".equalsIgnoreCase(type))
      {
         bnt = new NativeString();
         asPtr = true;
         
         if ("RETURN".equalsIgnoreCase(mode))
         {
            copyOut = false;
         }
      }
      else if ("LONG".equalsIgnoreCase(type))
      {
         bnt = new NativeSignedInt32();
      }
      else if ("UNSIGNED-LONG".equalsIgnoreCase(type))
      {
         bnt = new NativeUnsignedInt32();
      }
      else if ("SHORT".equalsIgnoreCase(type))
      {
         bnt = new NativeSignedInt16();
      }
      else if ("UNSIGNED-SHORT".equalsIgnoreCase(type))
      {
         bnt = new NativeUnsignedInt16();
      }
      else if ("INT64".equalsIgnoreCase(type))
      {
         bnt = new NativeSignedInt64();
      }
      else if ("BYTE".equalsIgnoreCase(type))
      {
         bnt = new NativeInt8();
      }
      else if ("DOUBLE".equalsIgnoreCase(type))
      {
         bnt = new NativeDouble();
      }
      else if ("FLOAT".equalsIgnoreCase(type))
      {
         bnt = new NativeFloat();
      }
      else if ("MEMPTR".equalsIgnoreCase(type))
      {
         bnt = new NativeBuffer();
         asPtr = true;
      }
      
      // avoid NPE, we will fail later in inbound processing at the type checking logic
      if (bnt != null)
      {
         // configure the type
         bnt.setCopyIn(copyIn);
         bnt.setCopyOut(copyOut);
         bnt.setPassByPointer(asPtr);
      }
      
      return bnt;
   }
   
   /**
    * This drives the unknown, uninitialized and empty string checking for a given argument
    * and generates errors in a 4GL compatible manner.  Calls {@link #checkUnknownWorker}
    * for the core processing.
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    * @param    mode
    *           "INPUT", "INPUT-OUTPUT", "OUTPUT" or "RETURN".
    * @param    cls
    *           The data type of the argument that was passed, which may be an array.
    * @param    arg
    *           The actual argument instance that was passed, which may be an array.
    * @param    bnt
    *           The descriptor for this scalar parameter or for an array. Some cases of
    *           unknown value, uninitialized variable or empty string will cause the normal
    *           copy back behavior to be silently ignored instead of raising an error. This
    *           method will edit the descriptor(s) as needed based on the actual value passed.
    *
    * @return   <code>true</code> to cause a silent return from the calling method.
    */
   private static boolean checkUnknown(String         pname,
                                       String         iename,
                                       String         mode,
                                       Class<?>       cls,
                                       Object         arg,
                                       BaseNativeType bnt)
   {
      boolean array = cls.isArray();
      
      if (array)
      {
         Class<?> real  = cls.getComponentType();
         Object[] args  = (Object[]) arg;
         
         // if the bnt is not an array we have a serious issue; we also assume it will have
         // the same number of elements as our given variable
         NativeTypeArray  nta   = (NativeTypeArray) bnt;
         BaseNativeType[] descr = nta.getElements();
         
         for (int i = 0; i < args.length; i++)
         {
            if (checkUnknownWorker(pname, iename, mode, real, args[i], true, descr[i]))
            {
               return true;
            }
         }
         
         return false;
      }
      else
      {
         return checkUnknownWorker(pname, iename, mode, cls, arg, false, bnt);
      }
   }
   
   /**
    * This checks a single argument or a single argument array element to detect if it is
    * unknown or empty string and generate errors in a 4GL compatible manner.  Called by
    * {@link #checkUnknown}.
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    * @param    mode
    *           "INPUT", "INPUT-OUTPUT", "OUTPUT" or "RETURN".
    * @param    cls
    *           The data type of the argument that was passed, which may be an element of an
    *           array.
    * @param    arg
    *           The actual argument instance that was passed, which may be an element of an
    *           array.
    * @param    array
    *           <code>true</code> if this instance was an element of an array.
    * @param    bnt
    *           The descriptor for this scalar parameter (or array element). Some cases of
    *           unknown value, uninitialized variable or empty string will cause the normal
    *           copy back behavior to be silently ignored instead of raising an error. This
    *           method will edit the descriptor as needed based on the actual value passed.
    *
    * @return   <code>true</code> to cause a silent return from the calling method.
    */
   private static boolean checkUnknownWorker(String         pname,
                                             String         iename,
                                             String         mode,
                                             Class<?>       cls,
                                             Object         arg,
                                             boolean        array,
                                             BaseNativeType bnt)
   {
      boolean in    = mode.equals("INPUT");
      boolean inOut = mode.equals("INPUT-OUTPUT");
      boolean out   = mode.equals("OUTPUT");
      boolean ret   = mode.equals("RETURN");
      
      BaseDataType bdt = null;
      
      if (arg instanceof BaseDataType)
      {
         bdt = (BaseDataType) arg;
      }
      else
      {
         // anything else is going to fail downstream (as is done in the 4gl)
         return false;
      }
      
      if (cls.equals(integer.class) || cls.equals(int64.class))
      {
         if (bdt.isUnknown())
         {
            if ((in || inOut))
            {
               genInputUnknownErr(true);
               return true;
            }
            else if (out)
            {
               // int/int64 output mode with unknown calls API but doesn't copy back
               bnt.setCopyOut(false);
               
               // must force pointer mode becuase otherwise the parm won't render properly
               // on the client side (it won't render as a pointer)
               bnt.setPassByPointer(true);
            }
            
            // int/int64 return mode with unknown calls API and DOES copy back
         }
      }
      else if (cls.equals(decimal.class))
      {
         if (bdt.isUnknown())
         {
            if ((in || inOut))
            {
               genInputUnknownErr(true);
               return true;
            }
            
            // dec output mode and return mode with unknown calls API and DOES copy back
         }
      }
      else if (cls.equals(character.class))
      {
         if (bdt.isUnknown())
         {
            if (in || inOut)
            {
               genInputUnknownErr(true);
               return true;
            }
            else if (out || ret)
            {
               genCharUnknownErr(true);
               return true;
            }
         }
         else
         {
            // empty string case calls API but doesn't copy back
            if (TextOps.lengthOf((character) bdt) == 0)
            {
               bnt.setCopyOut(false);
            }
         }
      }
      else if (cls.equals(longchar.class))
      {
         if (bdt.isUnknown())
         {
            if (in || inOut)
            {
               if (array)
               {
                  genInputUnknownErr(true);
                  return true;
               }
               else
               {
                  // non-array elements have suppressable errors in this case 
                  genCharUnknownErr(false);
                  return true;
               }
            }
            else if (out)
            {
               // output elements have suppressable errors for the non-array case and
               // unsuppressable errors in an array
               genCharUnknownErr(array);
               return true;
            }
            else if (ret)
            {
               // return mode case calls API but doesn't copy back
               bnt.setCopyOut(false);
            }
         }
         else
         {
            if (TextOps.lengthOf((longchar) bdt) == 0)
            {
               if (in || inOut || out)
               {
                  genCharUnknownErr(false);
                  return true;
               }
               else if (ret)
               {
                  // empty string return case calls API but doesn't copy back
                  bnt.setCopyOut(false);
               }
            }
         }
      }
      else if (cls.equals(memptr.class))
      {
         if (bdt.isUnknown())
         {
            if (in || inOut)
            {
               genInputUnknownErr(true);
               return true;
            }
            
            // unknown value as an array element is passed as a null pointer but for
            // a scalar instance it raises an error
            if (out && !array)
            {
               genUninitMemptrErr(pname, iename);
               return true;
            }
         }
         else
         {
            // INPUT and INPUT-OUTPUT cases will be processed later (during value checking)
            
            // uninitialized/0-size as an array element is passed as a null pointer but for
            // a scalar instance it raises an error
            if (out && !array)
            {
               memptr ptr = (memptr) bdt;
               
               if (ptr.isUninitialized() || ptr.lengthOf() == 0)
               {
                  genUninitMemptrErr(pname, iename);
                  return true;
               }
            }
         }
         
         // return case that is uninitialized works fine (address is copied back)
         // return case that is unknown or 0 size actually abends 4gl (we will implement these
         // like the uninitialized case since the 4gl can't work)
      }
      
      // anything else is going to fail downstream (as is done in the 4gl)
      return false;
   }
   
   /**
    * This drives the data type and mode checking for a given argument and will generate errors
    * in a 4GL compatible manner.  First the RUN statement mode is compared against the defined
    * mode for the corresponding DEFINE PARAMETER in the PROCEDURE EXTERNAL statement.  Second,
    * the data type of the actual runtime (RUN statement) argument is compared with the defined
    * data type in the corresponding DEFINE PARAMETER in the PROCEDURE EXTERNAL statement. The
    * first failure found is raised as an error.
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    * @param    parm
    *           The parameter definition.
    * @param    runmode
    *           'I' for "INPUT", 'U' for "INPUT-OUTPUT" or 'O' for "OUTPUT". "RETURN" parameters
    *           are passed at runtime as "OUTPUT").
    * @param    cls
    *           The data type of the argument that was passed (or the type of the element of the
    *           array that was passed, which is the type to be considered.
    *
    * @return   <code>true</code> to cause a silent return from the calling method.
    */
   private static boolean checkModeAndType(String    pname,
                                           String    iename,
                                           Parameter parm,
                                           char      runmode,
                                           Class<?>  cls)
   {
      // RUN statement modes don't use RETURN, they are always specified as OUTPUT; for this
      // reason, we use an alternate form of the getter for the mode, which will return OUTPUT
      // for both OUTPUT and RETURN parms; this simplifies the logic below
      String mode = parm.getSignatureMode();
      
      // check our mode before checking the type
      if ((runmode == InternalEntry.INPUT_MODE        && !"INPUT".equals(mode))        ||
          (runmode == InternalEntry.INPUT_OUTPUT_MODE && !"INPUT-OUTPUT".equals(mode)) ||
          (runmode == InternalEntry.OUTPUT_MODE       && !"OUTPUT".equals(mode)))
      {
         genMismatchingModeErr(pname, iename);
         return true;
      }
      
      String  type    = parm.getType().toUpperCase();
      boolean typeErr = true;
      
      // now check the type
      if (cls.equals(integer.class) || cls.equals(int64.class))
      {
         if ("BYTE".equals(type)           ||
             "SHORT".equals(type)          ||
             "UNSIGNED-SHORT".equals(type) ||
             "LONG".equals(type)           ||
             "UNSIGNED-LONG".equals(type)  ||
             "INT64".equals(type))
         {
            typeErr = false;
         }
      }
      else if (cls.equals(decimal.class))
      {
         if ("FLOAT".equals(type) || "DOUBLE".equals(type))
         {
            typeErr = false;
         }
      }
      else if (cls.equals(character.class) || cls.equals(longchar.class))
      {
         if ("CHARACTER".equals(type))
         {
            typeErr = false;
         }
      }
      else if (cls.equals(memptr.class))
      {
         if ("MEMPTR".equals(type))
         {
            typeErr = false;
         }
      }
      
      if (typeErr)
      {
         genMismatchingTypeErr(pname, iename);
         return true;
      }
      
      return false;
   }
   
   /**
    * Check if the given argument (or if it is an array, if the argument's elements) is within a
    * valid range for the type specified. Any out of range value will raise an error (except in
    * silent error mode).
    * <p>
    * <pre>
    * BYTE: -128 through 255
    * SHORT: -32768 through 32767
    * UNSIGNED-SHORT: 0 through 65535
    * LONG: -2147483648 through 2147483647
    * UNSIGNED-LONG: 0 through 4294967295    
    * </pre>
    *
    * @param    type
    *           The library function's parameter type as defined in the DEFINE PARAMETER
    *           statement.
    * @param    cls
    *           The class of the argument, which may be an array type.
    * @param    arg
    *           The argment to be checked.
    *
    * @return   <code>true</code> if the value is out of bounds.
    */
   private static boolean checkIntegerRange(String type, Class<?> cls, Object arg)
   {
      // quick out if the library type is int64 since that cannot overflow or underflow
      if ("INT64".equals(type))
         return false;
      
      boolean array = cls.isArray();
      
      if (array)
      {
         Object[] args = (Object[]) arg;
         
         for (int i = 0; i < args.length; i++)
         {
            int64 value = (int64) args[i];
            
            if (checkIntegerRangeWorker(type, value.longValue()))
            {
               return true;
            }
         }
         
         return false;
      }
      else
      {
         int64 value = (int64) arg;
         return checkIntegerRangeWorker(type, value.longValue());
      }
   }
   
   /**
    * Check if the given value is within a valid range for the type specified. Any out of range
    * value will raise an error (except in silent error mode).
    * <p>
    * <pre>
    * BYTE: -128 through 255
    * SHORT: -32768 through 32767
    * UNSIGNED-SHORT: 0 through 65535
    * LONG: -2147483648 through 2147483647
    * UNSIGNED-LONG: 0 through 4294967295    
    * </pre>
    *
    * @param    type
    *           The library function's parameter type as defined in the DEFINE PARAMETER
    *           statement.
    * @param    value
    *           The value to be checked.
    *
    * @return   <code>true</code> if the value is out of bounds.
    */
   private static boolean checkIntegerRangeWorker(String type, long value)
   {
      if (("BYTE".equals(type)           && (value < -128 || value > 255))               ||
          ("SHORT".equals(type)          && (value < -32768 || value > 32767))           ||
          ("UNSIGNED-SHORT".equals(type) && (value < 0 || value > 65535))                ||
          ("LONG".equals(type)           && (value < -2147483648 || value > 2147483647)) ||
          ("UNSIGNED-LONG".equals(type)  && (value < 0 || value > 4294967295L)))
      {
         // TODO: does this error have the same flaw as 15747 where Long.MIN_VALUE will display
         // as "-(" instead of "-9223372036854775808"? 
         String spec = "Value %d does not fit in %s DLL datatype";
         ErrorManager.recordOrThrowError(13712,
                                         String.format(spec, value, type.toLowerCase()),
                                         false);
         return true;
      }
      
      return false;
   }
   
   /**
    * Copy the data for the given INPUT or INPUT-OUTPUT argument (or if it is an array, the
    * argument's elements) into the container for transport to the client.
    * <p>
    * OUTPUT parameters that are passed by pointer must also have their data copied in.
    *
    * @param    type
    *           The library function's parameter type as defined in the DEFINE PARAMETER
    *           statement.
    * @param    mode
    *           The parameter's mode (e.g. INPUT, INPUT-OUTPUT, OUTPUT, RETURN).
    * @param    cls
    *           The class of the argument, which may be an array type.
    * @param    arg
    *           The argment to be copied from.
    * @param    bnt
    *           The target to which the data will be copied.
    */
   private static void copyDataIn(String         type,
                                  String         mode,
                                  Class<?>       cls,
                                  Object         arg,
                                  BaseNativeType bnt)
   {
      boolean array  = cls.isArray();
      
      if (array)
      {
         Class<?> real = cls.getComponentType();
         Object[] args = (Object[]) arg;
         
         NativeTypeArray  nta = (NativeTypeArray) bnt;
         BaseNativeType[] elements = nta.getElements();
         
         for (int i = 0; i < args.length; i++)
         {
            copyDataInWorker(type, mode, real, args[i], elements[i], true);
         }
      }
      else
      {
         copyDataInWorker(type, mode, cls, arg, bnt, false);
      }
   }
   
   /**
    * Copy the data for the given argument into the container for transport to the client.
    *
    * @param    type
    *           The library function's parameter type as defined in the DEFINE PARAMETER
    *           statement.
    * @param    mode
    *           The parameter's mode (e.g. INPUT, INPUT-OUTPUT, OUTPUT, RETURN).
    * @param    cls
    *           The class of the argument.
    * @param    arg
    *           The argment to be copied from.
    * @param    bnt
    *           The target to which the data will be copied.
    * @param    array
    *           <code>true</code> if this is an array element.
    */
   private static void copyDataInWorker(String         type,
                                        String         mode,
                                        Class<?>       cls,
                                        Object         arg,
                                        BaseNativeType bnt,
                                        boolean        array)
   {
      if (cls.equals(integer.class) || cls.equals(int64.class))
      {
         int64         ival = (int64) arg;
         NativeInteger nint = (NativeInteger) bnt;
         
         // if this is for an output parm and it is unknown, we are marked as pass by
         // pointer and we must bypass the actual copy which would normally happen
         if (!ival.isUnknown())
         {
            long num = ival.longValue();
            
            // there is a strange 4GL quirk where SCALAR INPUT BYTE data is always passed
            // as a signed byte instead of optionally being unsigned (if the original value
            // was positive); this means that data that was passed in as a positive value
            // between 128 and 255 (inclusive) will be passed as a negative value
            if ("BYTE".equals(type) && "INPUT".equals(mode) && !array)
            {
               // the java byte is always a signed value; by casting to a byte and then
               // back to the long, we are naturally forcing a sign extension which exactly
               // matches the 4GL result (it doesn't change the binary representation of
               // the number it just sign extends what is there)
               num = (byte) num;
            }
            
            nint.setValue(num);
         }
      }
      else if (cls.equals(decimal.class))
      {
         if ("FLOAT".equals(type))
         {
            decimal     dval = (decimal) arg;
            NativeFloat nflt = (NativeFloat) bnt;
            BigDecimal  flt  = dval.toJavaType();
            
            nflt.setValue(flt.floatValue());
         }
         else if ("DOUBLE".equals(type))
         {
            decimal      dval = (decimal) arg;
            NativeDouble ndbl = (NativeDouble) bnt;
            
            ndbl.setValue(dval.doubleValue());
         }
      }
      else if (cls.equals(character.class))
      {
         character    ctxt = (character) arg;
         NativeString ntxt = (NativeString) bnt;
         
         ntxt.setValue(ctxt.toStringMessage());
      }
      else if (cls.equals(longchar.class))
      {
         longchar     ltxt = (longchar) arg;
         NativeString ntxt = (NativeString) bnt;
         
         ntxt.setValue(ltxt.toStringMessage());
      }
      else if (cls.equals(memptr.class))
      {
         memptr       ptr  = (memptr) arg;
         NativeBuffer nbuf = (NativeBuffer) bnt;
         
         nbuf.setValue(ptr);
      }
   }
   
   /**
    * Attempts to copy back all data (from the descriptors used for transport back to the
    * actual wrapper parameters passed in) that should be copied back, raising errors
    * to match the behavior of the 4GL when necessary.
    *
    * @param    defs
    *           Parameter definitions. This defines the required signature of the actual native
    *           API as defined by a converted 4GL <code>PROCEDURE EXTERNAL</code> statement. As
    *           such it may or may not match reality, but it is what the 4GL uses for error
    *           checking (and there isn't anything better that we can do).
    * @param    signature
    *           The class list represented by the caller's passed arguments (determined at 
    *           runtime). This is NOT the same as the actual signature of native API to be
    *           called. This will be compared to the native API's required signature and errors
    *           will be raised as needed.
    * @param    briefcase
    *           The set of return value and parameter descriptors which should be setup for
    *           transport to the client.  This is initially passed as an empty instance and
    *           if no errors are encountered, it will be fully configured for transport to the
    *           client on return.
    * @param    args
    *           Arguments that were passed (at runtime) to be the inputs to (and receive the
    *           outputs from) the API call.
    */
   private static void outboundProcessing(List<Parameter> defs,
                                          Class<?>[]      signature,
                                          Signature       briefcase,
                                          Object...       args)
   {
      int numDefs = defs.size();
      int offset  = 0;
      
      for (int i = 0; i < numDefs; i++)
      {
         Parameter      parm = defs.get(i);
         String         type = parm.getType();
         String         mode = parm.getMode();
         BaseNativeType bnt  = null;
         boolean        ret  = false;
         
         if ("RETURN".equalsIgnoreCase(mode))
         {
            bnt = briefcase.getReturnValue();
            ret = true;
            
            // compensate for the missing return parameter when looking up the BNT
            offset++;
         }
         else if ("INPUT-OUTPUT".equals(mode) || "OUTPUT".equals(mode))
         {
            bnt = briefcase.getArgument(i - offset);
         }
         else
         {
            // INPUT case bypasses the copying below
            continue;
         }
         
         if (!copyDataOut(type, signature[i], ret, bnt, args[i]))
         {
            // any failure (which didn't raise an error) causes us to return silently
            // without any further processing
            return;
         }
      }
   }
   
   /**
    * Copy the data for the given INPUT-OUTPUT, OUTPUT or RETURN descriptor (or if it is an
    * array, the array elements) into the argument.
    *
    * @param    type
    *           The library function's parameter type as defined in the DEFINE PARAMETER
    *           statement.
    * @param    cls
    *           The class of the argument, which may be an array type.
    * @param    ret
    *           <code>true</code> if this is a RETURN parameter.
    * @param    bnt
    *           The descriptor from which to copy.
    * @param    arg
    *           The target to which the data will be copied.
    *
    * @return   <code>true</code> if all processing occurred without error or <code>false</code>
    *           if further outbound processing should be silently (and immediately) aborted.
    */
   private static boolean copyDataOut(String         type, 
                                      Class<?>       cls,
                                      boolean        ret,
                                      BaseNativeType bnt,
                                      Object         arg)
   {
      boolean array = cls.isArray();
      boolean ok    = true;
      
      if (array)
      {
         Class<?> real = cls.getComponentType();
         Object[] args = (Object[]) arg;
         
         NativeTypeArray  nta = (NativeTypeArray) bnt;
         BaseNativeType[] elements = nta.getElements();
         
         for (int i = 0; i < args.length; i++)
         {
            if (!copyDataOutWorker(type, real, false, elements[i], args[i], true))
            {
               ok = false;
               break;
            }
         }
      }
      else
      {
         ok = copyDataOutWorker(type, cls, ret, bnt, arg, false);
      }
      
      return ok;
   }
   
   /**
    * Copy the data back into the given argument from the container for transport to the client.
    *
    * @param    type
    *           The library function's parameter type as defined in the DEFINE PARAMETER
    *           statement.
    * @param    cls
    *           The class of the argument.
    * @param    ret
    *           <code>true</code> if this is a RETURN parameter.
    * @param    bnt
    *           The descriptor from which to copy.
    * @param    arg
    *           The target to which the data will be copied.
    * @param    array
    *           <code>true</code> if this is related to the processing of an element in an
    *           array, <code>false</code> for a scalar variable.
    *
    * @return   <code>true</code> if all processing occurred without error or <code>false</code>
    *           if further outbound processing should be silently (and immediately) aborted.
    */
   private static boolean copyDataOutWorker(String         type,
                                            Class<?>       cls,
                                            boolean        ret,
                                            BaseNativeType bnt,
                                            Object         arg,
                                            boolean        array)
   {
      // certain cases that are expected to copy back are not supposed to, because
      // of other previously determined conditions
      if (!bnt.isCopyOut())
      {
         return true;
      }
      
      if (cls.equals(integer.class) || cls.equals(int64.class))
      {
         int64         ival = (int64) arg;
         NativeInteger nint = (NativeInteger) bnt;
         
         // TODO: when used for scalar (non-array) variables, the correct version of this error
         // would look like this:
         // Value <output_value> too large to fit in INTEGER. Line 0 in <called_procedure_name> <calling_program_name>. (15747)
         // But the current generated version will look like this for both scalar and array
         // cases (this is already correct for the array case):
         // Value <output_value> too large to fit in INTEGER. (15747)
         
         // may raise error 15747
         ival.assign(nint.getValue());
      }
      else if (cls.equals(decimal.class))
      {
         decimal dval = (decimal) arg;
         
         if ("FLOAT".equals(type))
         {
            NativeFloat nflt = (NativeFloat) bnt;
            
            // TODO: when the largest 32-bit floating point value (0x7f7fffff) is read into a
            // decimal, the 4GL will raise this error:
            // Insufficient space for float to decimal conversion. (86)
            // There is no good reason for this, since a 32-bit float can hold a value with an
            // exponent of 2^127 which is roughly the same as 10^38.5. In other words, a 32-bit
            // float should have a non-fractional maximum of 39 to 40 digits. Such a value is
            // SUPPOSED to fit into a 4GL decimal. We don't yet know enough about the other
            // possible failing cases to reliably implement this error behavior.
            
            dval.assign(nflt.getValue());
         }
         else if ("DOUBLE".equals(type))
         {
            NativeDouble ndbl = (NativeDouble) bnt;
            
            double newval = ndbl.getValue(); 
            
            // overflows will normally raise error 536, but when processing decimal array
            // elements the 4GL has a stange behavior where it will not raise an error
            // (regardless of whether NO-ERROR is used), instead it just displays the message
            // (if NO-ERROR is not used) and moves on; it does not abort the copying but of
            // course nothing will be copied to the element being processed in this case
            if (array)
            {
               // TODO: for some reason it is possible to get messages for errors 86 and even
               // 15747 (an integer error! but in this case there is no text, just the error
               // num) in this same array processing; normally these are not possible but they
               // are here; the problem is we have no idea what actually causes these extra
               // messages so at this time we won't duplicate this behavior 
               
               if (dval.isOverflow(new BigDecimal(newval), false, false))
               {
                  if (!ErrorManager.isSilent())
                  {
                     ErrorManager.displayError(536, "Decimal number is too large");
                  }
                  
                  // don't abort the continued output processing
                  return true;
               }
            }
            
            // may raise error 536 in the non-array case (which is OK)
            dval.assign(newval);
         }
      }
      else if (cls.equals(character.class) || cls.equals(longchar.class))
      {
         Text         ctxt = (Text) arg;
         int          clen = TextOps.lengthOf(ctxt);
         NativeString ntxt = (NativeString) bnt;
         String       out  = ntxt.getValue();
         
         if (out == null)
         {
            // this should not be able to happen from converted 4GL code
            throw new IllegalStateException("NativeString null value was found!");
         }
         
         if (out.length() > clen)
         {
            // this is a 4GL limitation that we will maintain, even though we can easily
            // copy the data back anyway without any corruption
            String msg = String.format("Your %s buffer has been overrun.  Memory has been " +
                                       "corrupted",
                                       cls.equals(character.class) ? "CHARACTER" : "LONGCHAR"); 
            ErrorManager.recordOrThrowError(14528, msg, false);
         }
         else
         {
            ctxt.assign(out);
         }                                                                    
      }
      else if (cls.equals(memptr.class))
      {
         // only return parameters need any processing since all other types are just edits to
         // the already existing buffer (and if there is no existing buffer then it would have
         // failed during inbound processing)
         if (ret)
         {
            memptr       ptr  = (memptr) arg;
            NativeBuffer nbuf = (NativeBuffer) bnt;
            memptr       val  = nbuf.getValue();
            
            if (val != null && !val.isUnknown())
            {
               // technically, the 4GL would abend (not a 4GL error, the process actually traps)
               // if the return memptr passed by the caller was unknown or 0-sized (although 
               // passing an uninitialized memptr works fine); but we will just treat all those
               // cases the same
               ptr.setPointerValue(val.getPointerValue());
            }
         }
      }
      
      return !ErrorManager.isPendingError();
   }
   
   /**
    * Generate an indeterminate array parameter error.
    */
   private static void genIndeterminateArrayErr()
   {
      String msg = "You cannot pass an indeterminate length array to a DLL as INPUT or receive " +
                   "one back as OUTPUT";
      ErrorManager.recordOrThrowError(12201, msg, false);
   }
   
   /**
    * Generate an cannot pass an array variable for a RETURN parameter error.
    */
   private static void genNoArrayForReturnErr()
   {
      String msg = "You cannot pass an array variable for the RETURN parameter of a DLL";
      ErrorManager.recordOrThrowError(12270, msg, false);
   }
   
   /**
    * Generate an character or longchar value cannot be unknown error.
    *
    * @param    force
    *           <code>true</code> to throw the error even in silent mode.
    */
   private static void genCharUnknownErr(boolean force)
   {
      String spec = "A CHARACTER or LONGCHAR variable passed as an OUTPUT parameter to a DLL " +
                    "cannot be the Unknown value or be uninitialised";
                    
      genForcableError(12450, spec, force); 
   }

   /**
    * Generate an error that can optionally not be suppressed by NO-ERROR.
    *
    * @param    num
    *           The 4GL error number.
    * @param    spec
    *           The 4GL error text.
    * @param    force
    *           <code>true</code> to throw the error even in silent mode.
    */
   private static void genForcableError(int num, String spec, boolean force)
   {
      try
      {
         ErrorManager.recordOrThrowError(num, spec, false);
      }
      
      catch (ErrorConditionException err)
      {
         // this should work so long as we are not in warning mode, which is expected to not
         // be active at this time
         if (force)
         {
            err.setForce(true);
         }
         
         throw err;
      }
   }

   /**
    * Generate an uninitialized MEMPTR error.
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    */
   private static void genUninitMemptrErr(String pname, String iename)
   {
      String spec   = "DLL procedure %s %s using an uninitialised MEMPTR";
      ErrorManager.recordOrThrowError(3233, String.format(spec, iename, pname), false);
   }
   
   /**
    * Generate a mismatching mode error.
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    */
   private static void genMismatchingModeErr(String pname, String iename)
   {
      String spec   = "Mismatched parameter types passed to procedure %s %s";
      ErrorManager.recordOrThrowError(3230, String.format(spec, iename, pname), false);
   }
   
   /**
    * Generate a mismatching type error.
    *
    * @param    pname
    *           The legacy name of the external procedure from which this API is being called.
    * @param    iename
    *           The internal-entry name being called which is usually the same as the entry
    *           point name in the library except where an ordinal is being used.
    */
   private static void genMismatchingTypeErr(String pname, String iename)
   {
      String spec   = "Mismatch in the parameter datatypes in DLL procedure %s %s";
      ErrorManager.recordOrThrowError(3231, String.format(spec, iename, pname), false);
   }
   
   /**
    * Generate a library load error.
    *
    * @param    libname
    *           The name of the library that failed to be loaded.
    * @param    platform
    *           A LibraryManager constant defining the platform on which the client was running.
    */
   private static void genLibraryLoadErr(String libname, int platform)
   {
      String msg = String.format("** Could not load DLL procedure %s. (%d)", libname, 3258);
      
      if (platform == LibraryManager.PLATFORM_WINDOWS)
      {
         ErrorManager.recordOrThrowError(3258, msg, false, true);
      }
      else
      {
         // one of the rare cases where UNIX/Linux is more complicated than Windows
         String[] msgs = new String[3];
         
         // we have to manually build the text since the 8013 and 8014 messages don't have a
         // trailing period
         msgs[0] = String.format("** Could not open Dynamic Library: %s (%d)", libname, 8013);
         msgs[1] = String.format("** DLL Error : %s: cannot open shared object file: " +
                                 "No such file or directory (%d)",
                                 libname, 8014);
         msgs[2] = msg;
         
         ErrorManager.recordOrThrowError(new int[]{ 8013, 8014, 3258 }, msgs, false, true);
      }
   }
   
   /**
    * Generate a missing entrypoint error.
    *
    * @param    funcname
    *           The name of the function that failed to be loaded.
    * @param    ordinal
    *           The ordinal of the function that failed to be loaded or -1 if no ordinal was
    *           specified.
    */
   private static void genEntrypointErr(String funcname, int ordinal)
   {
      // yes, in the 4GL this funky behavior happens when an ordinal is specified even on
      // non-Windows platforms
      
      String descr = (ordinal == -1) ? funcname : String.format("%d", ordinal);
      String msg   = String.format("Could not find the entrypoint %s", descr);
      
      ErrorManager.recordOrThrowError((ordinal == -1) ? 3260 : 3259, msg, false);
   }
   
   /**
    * Container for context-local data.
    */
   private static class WorkArea
   {
      /** Remote proxy for access to and management of native libraries. */
      private NativeAPICaller libmgr = null;
   }
}