# raw.java

    /*
    ** Module   : raw.java
    ** Abstract : Progress 4GL compatible raw data type
    **
    ** Copyright (c) 2005-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ----------------------------Description-----------------------------
    ** 001 GES 20050906   @22563 Created initial version with support for a subset of raw processing.
    ** 002 ECF 20060403   @25321 Added copy constructor.
    ** 003 GES 20060724   @28173 Use forced assign in copy constructor.
    ** 004 GES 20090422   @41913 Converted to standard string formatting.
    ** 005 CA  20130123          Added c'tor which builds a new instance from a BaseDataType value,
    **                           if possible. This should be used only when constructing an instance
    **                           using the value returned by a DYNAMIC-FUNCTION call; 4GL does some
    **                           special casting in this case, which is not implemented yet.
    ** 006 GES 20130322          Fixed the BDT c'tor.
    ** 007 OM  20130329          Added instantiateUnknownRaw() static method.
    ** 008 GES 20130523          Major rewrite to implement a nearly complete implementation. Many
    **                           deviations from the 4GL were found and fixed. The actual data access
    **                           and storage is now in this class instead of in the parent, and this
    **                           is exposed via a set of abstract worker methods that we implement
    **                           here.
    ** 009 GES 20131212          Made isUninitialized() public.
    ** 010 MAG 20140602          Fixed the BDT c'tor and modified the assign(BDT, boolean) on _POLY
    **                           morphing. Add compareTo(Object).
    ** 011 SVL 20140709          Removed debug output.
    ** 012 CA  20140811          Fixed clearing of unknown flag in assignment/output param.
    ** 013 ECF 20150715          Replace StringBuffer with StringBuilder.
    ** 014 OM  20160711          Update to match the BinaryData interface related to buffer length.
    ** 015 CA  20160627          Reworked undoable support - the undoables register themselves with
    **                           all blocks up the stack, until either 1. the tx block which created
    **                           it is reached or 2. the last tx block where it was saved is reached.
    ** 016 OM  20160912          Small optimisations. Javadoc updates.
    ** 017 GES 20171207          Removed forced version of assign().
    ** 018 CA  20181128          Fixed return type of instantiateUnknownRaw.
    ** 019 IAS 20190617          Changed the writeByteRange method
    ** 020 CA  20190728          Fixed writeByteRange.
    ** 021 OM  20200507          Handling empty strings in constructor as a unknown value.
    ** 022 IAS 20200908          Rework (de)serialization.
    ** 023 IAS 20201007          Added Type enum
    ** 024 ME  20210504          Handle character assign, check dynamic data type conversion allowed.
    **     CA  20210818          Fixed undoable support - the changed value was being saved, instead of the
    **                           'before' byte array.
    **     AL2 20220319          Added value proxy check in assign.
    ** 025 CA  20230215          'instantiateDefault' now is implemented by each sub-class.
    **     CA  20230215          'duplicate()' method returns the real type instead of BDT.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util;

    import static com.goldencode.util.NativeTypeSerializer.*;
    import java.io.*;
    import java.util.*;

    import org.apache.commons.codec.binary.Base64;

    import com.goldencode.p2j.util.BaseDataType.*;

    /**
     * A class that implements common services for a Progress 4GL compatible raw data type whose data
     * is mutable.  Only a subset of the Progress language features are supported.
     * <p>
     * Format string related processing is handled in method {@link #toString(String)}.
     * <p>
     * All {@code RAW} features other than string formatting are implemented in the common base class
     * {@link BinaryData} which is shared with {@code MEMPTR} types.
     *<p>
     * In Progress 4GL, the actual maximum size of a {@code RAW} cannot be reliably
     * determined.  The largest value that has been found to consistently work is 31990. But
     * supposedly up to 32000 should work.  It turns out these don't work in practice. This
     * implementation works to the 32000 size. Although the documentation suggests that 32kb
     * (32768 bytes or 32 times 1024) is the limit, attempting to use values of that size
     * actually cause memory access violations and abends, regardless of what the 4GL
     * documentation states. This class does NOT duplicate the instability or 4GL process
     * abends, but is otherwise compatible.
     */
    public class raw
    extends BinaryData
    {
       /** The maximum size of the binary data stored in an instance. */
       public static final int MAX_SIZE = 32000;

       /**
        * Stores the state of this instance as an array of bytes.  This is independent of the state
        * of whether or not this instance represents {@code unknown}.
        */
       private byte[] value = null;

       /**
        * Default constructor, creates an instance that represents a 0 length byte array.
        */
       public raw()
       {
          value = new byte[0];
       }

       /**
        * This is a special c'tor which should be used only when converting the
        * value returned by a function or method with polymorphic return type into the
        * expected type (i.e. DYNAMIC-FUNCTION()).  In such cases, the 4GL does some
        * automatic type conversion (see {@link #assign(BaseDataType)}).
        *
        * @param    value
        *           The value to be used for this instance.
        */
       public raw(BaseDataType value)
       {
          // the value may be a proxy; make sure to unwrap and work with the real value
          value = value == null ? null : value.val();
          if (value == null || value.isUnknown())
          {
             setUnknown();
          }
          else
          {
             if (value instanceof Text)
             {
                // FIXME Is any valid value?
                // Why P4G show this message for character and longchar?
                String err = "Invalid encoded representation of RAW data.";
                ErrorManager.recordOrThrowError(4957, err);
             }
             else
             {
                assign(value);
             }
          }
       }

       /**
        * Compares this instance with the specified instance and returns a -1
        * if this instance is less than the specified, 0 if the two instances
        * are equal and 1 if this instance is greater than the specified
        * instance.  This is the implementation of the {@link Comparable} interface.
        * <p>
        * The algorithm will fail to give meaningful results in the case where
        * one tries to sort against other objects that do not represent compatible values.
        *
        * @param   obj
        *          The instance to compare against.
        *
        * @return  -1, 0 or 1 depending on if this instance is less than, equal to or greater
        *          (respectively) than the specified instance {@code obj}.
        */
       @Override
       public int compareTo(Object obj)
       {
          if (obj instanceof BaseDataType)
          {
             return super.compareTo(new raw((BaseDataType) obj));
          }

          return super.compareTo(obj);
       }

       /**
        * Constructs an instance after copying the parameter's data into the
        * internal representation of this class.
        * <p>
        * If the parameter's represents {@code unknown}, this instance will also represent
        * {@code unknown}.
        *
        * @param   value
        *          The value to be used for this instance.
        */
       public raw(raw value)
       {
          assign(value);
       }

       /**
        * Constructor which creates an instance which is initialized based on the given binary data.
        *
        * @param    val
        *           The binary data to be used as the contents or if {@code null} then the
        *           instance will represent {@code unknown}.
        */
       public raw(byte[] val)
       {
          super(val);
          replaceContents(Arrays.copyOf(val, val.length));
       }

       /**
        * Constructs an instance which converts the contents of the given
        * Progress 4GL compatible string form of a raw value into the proper
        * binary form.  When Progress converts a raw value into a string,
        * it inserts 6 hexadecimal characters at the front of the string and
        * then formats all other data as a single Base64 encoded value (it
        * is NOT split into 76 character line segments).  The first 2 hexadecimal
        * characters are always {@code 02} and the following 4 hexadecimal
        * characters are a hexadecimal representation of the length of the
        * resulting value.  These 6 preceding bytes are simply dropped.
        *
        * @param    value
        *           The Base64 value to be converted into the resulting
        *           instance binary data.
        */
       public raw(String value)
       {
          if (value == null)
          {
             setUnknown();
          }
          else if (value.isEmpty())
          {
             this.value = new byte[0]; // same as the default c'tor
          }
          else
          {
             replaceContents(Base64.decodeBase64(value.substring(6).getBytes()));
          }
       }

       /**
        * Constructs an instance which converts the contents of the given
        * Progress 4GL compatible string form of a raw value into the proper
        * binary form.  When Progress converts a raw value into a string,
        * it inserts 6 hexadecimal characters at the front of the string and
        * then formats all other data as a single Base64 encoded value (it
        * is NOT split into 76 character line segments).  The first 2 hexadecimal
        * characters are always {@code 02} and the following 4 hexadecimal
        * characters are a hexadecimal representation of the length of the
        * resulting value.  These 6 preceding bytes are simply dropped.
        *
        * @param    value
        *           The Base64 value to be converted into the resulting instance binary data. If this
        *           is the {@code unknown}, then the result is the {@code unknown}.
        */
       public raw(character value)
       {
          if (value == null || value.isUnknown())
          {
             setUnknown();
          }
          else
          {
             String str = value.getValue().substring(6);
             replaceContents(Base64.decodeBase64(str.getBytes()));
          }
       }

       /**
        * Creates a new instance of the same type that represents {@code unknown}.
        *
        * @return   An instance that represents {@code unknown}.
        */
       public static raw instantiateUnknownRaw()
       {
          raw r = new raw();
          r.setUnknown();

          return r;
       }

       /**
        * Get the type
        * @return type
        */
       public Type getType()
       {
          return Type.RAW;
       }

       /**
        * Does the same as standard {@link #clone()} method but returns an instance of
        * {@link BaseDataType} and doesn't throw the {@link CloneNotSupportedException}.
        *
        * @return   A clone of this instance.
        */
       public raw duplicate()
       {
          raw r = new raw();

          // set the instance data
          r.assign(this);

          return r;
       }

       /**
        * Creates a new instance of the same type that represents {@code unknown}.
        *
        * @return   An instance that represents {@code unknown}.
        */
       public BaseDataType instantiateUnknown()
       {
          return raw.instantiateUnknownRaw();
       }

       /**
        * Creates a new instance of the same type that represents the default initialized value.
        *
        * @return   An instance that represents the default value.
        */
       @Override
       public BaseDataType instantiateDefault()
       {
          return new raw();
       }

       /**
        * Reports if this instance represents an uninitialized buffer. Being uninitialized is not the
        * same as being of zero length.  Rather it means that the buffer was never allocated. Being
        * {@code unknown} is also different from being uninitialized.
        *
        * @return    {@code true} if this instance is uninitialized.
        */
       public boolean isUninitialized()
       {
          return (value == null);
       }

       /**
        * Creates a default string representation of the instance data which will be equivalent to the
        * result of {@link #toStringExport}.  If the instance represents {@code unknown}, a
        * {@code '?'} will be returned.
        *
        * @return   The formatted string.
        */
       public String toString()
       {
          return toString(null);
       }

       /**
        * Creates a string representation of the instance data using the given format string.
        * A {@code null} format string is equivalent to the result of {@link #toStringExport}.
        * If the instance is {@code unknown}, a '?' will be returned.
        *
        * @param    fmt
        *           The format string to use.
        *
        * @return   The formatted string.
        */
       public String toString(String fmt)
       {
          if (fmt == null)
          {
             return toStringExport();
          }

          if (unknown)
          {
             return "?";
          }

          if (fmt.length() == 0)
          {
             ErrorManager.recordOrThrowError(74, "Value  cannot be displayed using ");
             return "";
          }

          // use the normal character type to handle this data
          character tmp = new character(new String(value));

          return tmp.toString(fmt);
       }

       /**
        * Creates a string representation of the instance data in a form that is compatible with the
        * {@code MESSAGE} language statement.  If the instance represents {@code unknown}, a
        * {@code '?'} will be returned.
        *
        * @return   The 'message' formatted string.
        */
       public String toStringMessage()
       {
          return toStringExport();
       }

       /**
        * Creates a string representation of the instance data using the 'export'
        * format in which the data is written as a base64 string prepended with 6
        * hexadecimal bytes of "header" information. Warning: this method is
        * hard-coded to produce a header string starting with "02" (this seems
        * to be a fixed value for normal {@code raw} data in Progress, but
        * it may be different when records or other structured data is written)
        * and this is followed by 4 bytes specifying the length of the data.
        * <p>
        * If the instance represents {@code unknown}, a '?' will be returned.
        *
        * @return   The 'export' formatted string.
        */
       public String toStringExport()
       {
          // quick out
          if (unknown)
          {
             return "?";
          }

          // always starts with 0x02
          StringBuilder sb = new StringBuilder("02");

          // add length as 4 hexadecimal chars
          sb.append(String.format("%04X", value.length));

          if (value.length > 0)
          {
             // add the contents of the array encoded base64
             sb.append(new String(Base64.encodeBase64(value)));
          }

          return sb.toString();
       }

       /**
        * Sets the array to a given length, which may result in either extending
        * or truncating the array.
        *
        * @param    len
        *           New length of the array.
        *
        * @return   The current instance is returned.
        */
       public BinaryData setLength(long len)
       {
          extendBytes(len);
          return this;
       }

       /**
        * Replacement for the default object reading method. The latest
        * state is read from the input source.
        *
        * @param    in
        *           The input source from which fields will be restored.
        *
        * @throws   IOException
        *           In case of I/O errors.
        * @throws   ClassNotFoundException
        *           If payload can't be instantiated.
        */
       public void readExternal(ObjectInput in)
       throws IOException,
              ClassNotFoundException
       {
          super.readExternal(in);
          value = readByteArray(in);
       }

       /**
        * Replacement for the default object writing method. The latest
        * state is written to the output destination.
        *
        * @param    out
        *           The output destination to which fields will be saved.
        *
        * @throws   IOException
        *           In case of I/O errors.
        */
       public void writeExternal(ObjectOutput out)
       throws IOException
       {
          super.writeExternal(out);
          writeByteArray(out, value);
       }

       /**
        * Sets the state of this instance's {@code unknown} flag to {@code true}.
        * <p>
        * <b>Warning: the data stored in this instance WILL BE deallocated after calling this
        * method.</b>
        */
       public void setUnknown()
       {
          if (!unknown)
          {
             checkUndoable(true);
          }

          super.setUnknown();
       }

       /**
        * Sets the state of this instance's {@code unknown} flag to {@code false}.
        */
       @Override
       protected void clearUnknown()
       {
          if (unknown)
          {
             checkUndoable(true);
          }

          super.clearUnknown();
       }

       /**
        * Release any resources associated with the instance.
        */
       protected void deallocate()
       {
          checkUndoable(() -> new raw(new byte[0]));

          value = new byte[0];
       }

       /**
        * Determines if the array of binary data stored in an instance of this
        * class must be limited in length.
        *
        * @return    {@code true} if the length must be limited.
        */
       protected boolean isLengthLimited()
       {
          return true;
       }

       /**
        * Determines the maximum length of the array of binary data stored in an
        * instance of this class.
        *
        * @return    Always MAX_SIZE (32KB).
        */
       protected long lengthLimit()
       {
          return MAX_SIZE;
       }

       /**
        * Determines if the array of binary data stored in an instance of this
        * class must be automatically extended in length when operations
        * write past the current length.  Such extensions will pad with
        * {@code null} bytes.
        *
        * @return    {@code true} if the length must automatically extended.
        */
       protected boolean isAutoExtend()
       {
          return true;
       }

       /**
        * Replace the entire contents of the current buffer with the given array.
        * <p>
        * The {@link BinaryData#unknown} flag is also cleared, only if {@code value} is not
        * {@code null} and if the length is greater than 0.
        *
        * @param    value
        *           The new contents.
        */
       protected void replaceContents(byte[] value)
       {
          checkUndoable(() -> new raw(value));

          this.value = value;

          if (value != null && value.length > 0)
          {
             clearUnknown();
          }
       }

       /**
        * Return the entire contents of the current buffer as an array. This may be a copy
        * of the data, depending on the subclass' implementation.  DO NOT MODIFY the data
        * that is returned, since it is undefined as to whether the changes will or will not
        * be reflected in the actual storage.
        *
        * @return    The contents of the instance (possibly a copy). If the instance is uninitialized
        *            the returned value will be a 0 length array. If the instance is {@code unknown}
        *            or has an undefined size, then {@code null} will be returned.
        */
       protected byte[] asByteArray()
       {
          return unknown ? null : value;
       }

       /**
        * Reports the current length of the buffer.
        *
        * @return  The length or 0 if uninitialized.
        */
       @Override
       protected long visibleLength()
       {
          return internalLength();
       }

       /**
        * Reports the current length of the buffer.
        *
        * @return  The length or 0 if uninitialized.
        */
       @Override
       protected long internalLength()
       {
          return (value == null) ? 0 : value.length;
       }

       /**
        * Returns the next index position into the array which is the {@code null} byte,
        * starting at a given index position.
        *
        * @param    pos
        *           The 0-based index into the array at which to start searching.
        *
        * @return   The next index into the array which is the {@code null} byte or the length
        *           of the array if no {@code null} byte exists.
        */
       protected long findNextNull(long pos)
       {
          // arrays can only have 32-bit indexes in the Java Language Spec
          int i = (int) pos;

          // not all instances will end in null byte, so check length first to prevent overrun
          while (i < value.length && value[i] != '\0')
          {
             i++;
          }

          return i;
       }

       /**
        * Helper to create a new byte array with the same contents as the given byte array and the
        * specified length.  If the given target length is smaller than the source array length, the
        * output array is truncated to the specified size.  If the target length is larger than the
        * source array length, the output array is extended to the given size and any additional
        * bytes are initialized to '\0'.
        * <p>
        * This is safe to use on 0 length arrays.
        * <p>
        * If the instance is {@code unknown}, this silently returns without making any changes.
        *
        * @param    len
        *           The output array length.  Negative values are silently ignored.  Positive values
        *           up to {@link #MAX_SIZE} are accepted. Values greater than
        *           {@link #MAX_SIZE} will cause an error condition to be raised.
        */
       protected void extendBytes(long len)
       {
          if (unknown || len < 0)
          {
             return;
          }

          if (len > MAX_SIZE)
          {
             genLimitedLengthError();
             return;
          }

          byte[] newVal;

          if (value == null)
          {
             newVal = new byte[(int) len];
          }
          else
          {
             newVal = Arrays.copyOf(value, (int) len);
          }

          checkUndoable(() -> new raw(newVal));

          value = newVal;
       }

       /**
        * Read the byte at the given location.  All boundary checking and error handling must have
        * already been done in the caller such that this is known to be a valid access. Any
        * unexpected failure may cause a platform-specific exception that terminates the process!
        *
        * @param    pos
        *           The 0-based index position at which to read the byte.
        *
        * @return   The byte at that location.
        */
       protected byte readByte(long pos)
       {
          return value[(int) pos];
       }

       /**
        * Write the byte into the given location. All boundary checking and error handling must have
        * already been done in the caller such that this is known to be a valid access. Any
        * unexpected failure may cause a platform-specific exception that terminates the process!
        *
        * @param    val
        *           The byte to write.
        * @param    pos
        *           The 0-based index position at which to read the byte.
        */
       protected void writeByte(byte val, long pos)
       {
          if (value[(int) pos] != val)
          {
             checkUndoable(true);
          }

          value[(int) pos] = val;
       }

       /**
        * Read the bytes at the given location.  All boundary checking and error handling must have
        * already been done in the caller such that this is known to be a valid access. Any
        * unexpected failure may cause a platform-specific exception that terminates the process!
        *
        * @param    pos
        *           The 0-based index position at which to read the bytes.
        * @param    len
        *           The number of bytes to read.
        * @param    endian
        *           {@code true} to force the returned data to be interpreted by the endian-ness
        *           of the native platform.  The returned data will be in little endian order if this
        *           flag is set (but on a big endian platform the bytes will be reversed first).
        *
        * @return   The byte range at that location.
        */
       protected byte[] readByteRange(long pos, long len, boolean endian)
       {
          // create buffer to contain the data
          byte[] data = new byte[(int) len];

          // find the ending index; array indexes are limited to 32-bit in the Java Language Spec
          int idx = (int) pos;
          int end = idx + (int) len;

          // copy the data in (may copy nothing if end == pos)
          System.arraycopy(value, idx, data, 0, end - idx);

          // we always return data in little endian order when the endian flag is set
          if (endian && !memptr.isLittleEndianPlatform(null))
          {
             data = reverseBytes(data);
          }

          return data;
       }

       /**
        * Write the bytes at the given location.  All boundary checking and error handling must have
        * already been done in the caller such that this is known to be a valid access. Any
        * unexpected failure may cause a platform-specific exception that terminates the process!
        *
        * @param    data
        *           The bytes to write.
        * @param    pos
        *           The 0-based index position at which to start writing the bytes.
        * @param    len
        *           How many bytes should be written. -1 means write everything to the end of the
        *           array.
        * @param    endian
        *           {@code true} to honor the endian-ness of the platform. The {@code data}
        *           array MUST be ordered in little-endian order if the endian flag is on.
        */
       protected void writeByteRange(byte[] data, long pos, long len, boolean endian)
       {
          // the data is always passed into us in little endian order when the endian flag is set
          if (endian && !memptr.isLittleEndianPlatform(null))
          {
             data = reverseBytes(data);
          }

          // find the ending index; array indexes are limited to 32-bit in the Java Language Spec
          int idx = (int) pos;
          int end = Math.min(idx + (int)(len == -1 ? data.length : len), value.length);

          boolean changed = false;

          // check the data if has changed
          for (int i = idx; i < end; i++)
          {
             byte newVal = data[i - idx];

             if (!changed && value[i] != newVal)
             {
                changed = true;
                break;
             }
          }

          if (changed)
          {
             checkUndoable(true);
          }

          // copy the data in
          for (int i = idx; i < end; i++)
          {
             value[i] = data[i - idx];
          }
       }

       @Override
       protected boolean isIncompatibleTypesOnConversion(BaseDataType value)
       {
          // invalid conversion is being thrown even if unknown
          if (value instanceof clob || value instanceof blob || value instanceof longchar)
          {
             incompatibleTypesOnConversion();
          }

          // unknown or empty string assignments are not performed
          if (value instanceof character && TextOps.isEmpty((character) value))
             return true;

          return super.isIncompatibleTypesOnConversion(value);
       }

       @Override
       public void assign(BaseDataType data)
       {
          if (data != null)
          {
             data = data.val();
          }

          if (data instanceof character && !data.isUnknown())
          {
             // first two characters should be "02"
             // following 4 are length of the initial raw "encoded" as string (hex)
             // the rest is base64 encoded value of initial raw
             String value = ((character) data).toStringMessage();

             if (value.startsWith("02") && value.length() >= 6)
             {
                assign(Base64.decodeBase64(value.substring(6).getBytes()));
             }
             else
             {
                ErrorManager.recordOrThrowError(4957, "Invalid encoded representation of RAW data");
             }
          }
          else
          {
             super.assign(data);
          }
       }
    }