# FileStream.java

    /*
    ** Module   : FileStream.java
    ** Abstract : represents a file or device for reading/writing
    **
    ** Copyright (c) 2005-2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
    ** 001 GES 20050712   @21743 Created initial version which provides input and output semantics
    **                           for any file-like file system object (file or device).
    ** 002 GES 20050801   @21939 New assign() interface for support of shared streams.
    ** 003 GES 20050824   @22212 Added error condition support.
    ** 004 GES 20060217   @24671 Added writeByte().
    ** 005 GES 20060225   @24739 Added call to cleanup() in close().
    ** 006 GES 20060525   @26517 Avoid setLength() on a device during init.
    ** 007 GES 20060905   @29227 Fix for readCh() to properly handle EOF.
    ** 008 NVS 20070725   @34685 Implemented methods isIn() and isOut(), which are newly added to the
    **                           base class as abstract.
    ** 009 SIY 20070928   @35318 Fix for PUT/DISPLAY handling regressions. Ref: #35314
    ** 010 GES 20080414   @38015 Support standard character conversion (just the default behavior,
    **                           no user-definable source and/or target codepages yet). This
    **                           allows other languages (and the upper ASCII range in the latin
    **                           codepages) to be properly handled. Added support for the no-convert
    **                           option as well, to disable this new support.
    ** 011 GES 20080724   @39235 Major rewrite to use NIO and memory mapped file support to massively
    **                           improve perf.
    ** 012 ECF 20090526   @42464 Fixed long file defect. An offset >2GB passed to map() was being
    **                           truncated when mapped to a MappedByteBuffer.
    ** 013 SVL 20110403          write(byte[]) was made public.
    ** 014 EVL 20121204          readLn() has been changed to use indexOfNewLine() to detect the line
    **                           separators. This allows to properly handle the input files coming
    **                           from other OS than one the P2J runtime is running.
    ** 015 OM  20130607          Made file and map(long) protected to be accessible in derived classes.
    ** 016 EVK 20140523          Added unbuffered stream operation.
    ** 017 OM  20170613          Added non-binary encoding for Windows streams. Implemented peekCh()
    **                           method.
    ** 018 OM  20171026          Removed ChUI dependency.
    ** 019 ECF 20171026          Added write(byte[], int, int) method.
    ** 020 EVL 20180620          Adding pulse on close for empty frames.
    ** 021 CA  20190905          Allow byte buffer instead of memory buffers for read-only files.
    ** 022 CA  20200116          Javadoc fixes.
    ** 023 IAS 20201219          Added codepage change at runtime. Javadoc fixes.
    ** 024 ME  20210322          Flush data/buffered when flush method is called.
    **     OM  20210328          OE quirk: if last line is not EOL terminated, the readLn() method returns the
    **                           previous read line (actually, if that EOL is missing the file is binary by
    **                           definition).
    **     OM  20210401          Reverting previous commit. It seems it is not the correct way to handle it.
    **     OM  20210526          Added support for conversion of stream to a character stream using a multibyte
    **                           enabled character set.
    **     OM  20210602          Improved detection of file encoding. Enabled multibyte support.
    **     OM  20210702          Added support for mapping invalid bytes detected during reading of input files.
    **                           Handled multibyte character spanning between two read byte batches.
    **     CA  20210709          The stream must work with bytes explicitly (for available, position).  Characters
    **                           are decoded only when they are explicitly request via readCh() and readChunk().
    **     CA  20210711          readCharacterWorker() tries to decodes bytes until a character is decoded.  The
    **                           assumption is that for any i.e. [b1, b2, b3] bytes which can decode a character,
    **                           [b1] and [b1, b2] can not be decoded as a character.
    **     CA  20210920          After decoding, the CharBuffer needs to be used via 'toString()', and not via
    **                           'array()', as that returns the internal array which may have additional elements.
    **     CA  20211001          Do not flush file resources on PUT/EXPORT statements.
    **     AL2 20211217          Make read bytes unsigned; -1 should be made signed and not represent EOF.
    **     EVL 20220325          Base refactoring for Stream based classes getting single byte and array of bytes.
    **     EVL 20220328          Reworked to be used as base class for CachedInputStream.
    **     EVL 20220330          Fixed regression for stream reset/reusage.
    **     EVL 20220401          Fixed additional regressions from refactorng.
    **                           Added more protection from underflow exception in certain class implementation.
    **     EVL 20220406          When we have no new line character inside single line file we have to return
    **                           empty string instead of the line content.  This is the OE 'feature' we need to
    **                           duplicate.
    **     TJD 20220331          Removed legacy ThreadLocalCoders, decoder for charset new instance is created
    **                           for every call
    **     OM  20220602          Invalid bytes are dropped if mapped to null.
    **     EVL 20221212          Adding LOB-DIR runtime support.
    **     EVL 20221222          Adding export approach implementation for LOB fields.
    ** 025 EVL 20230330          Fixed BOM header skipping for text file read operations.  Using seek statement
    **                           resets BOM consideration while seek function should consider BOM length.
    ** 026 HC  20230427          Implemented server-side file-system resource. File operations
    **                           performed on client can be optionally performed directly on server
    **                           without the expensive remote call.
    ** 027 AD  20230920          Fixed issue where blob type would not correctly write to file.
    **         20230922          Fixed similar problem happening with writing clob to file.
    **         20230926          Added length check for writeClob(), formated some comments.
    ** 028 GBB 20231213          Adding flag hasDuplicatedLastLine.
    ** 029 SB  20240617          Use isWindows instead of recalling EnvinronmentOps.isUnderWindows.
    ** 030 SP  20240603          Added support for "** <path> is not a directory (302)" error message.
    ** 031 GBB 20240912          Make two of the constructors public to allow access from FileSystemDaemon.
    ** 032 PMP 20241008          Removed outdated check related to old 4GL behavior.
    ** 033 PMP 20241009          Commented out-dated 4GL behavior.
    ** 034 SP  20250207          Refactored BOM encoding detection into the static method detectBOM(byte[]).
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util;

    import com.goldencode.p2j.security.*;

    import java.io.*;
    import java.nio.*;
    import java.nio.channels.*;
    import java.nio.charset.*;
    import java.nio.charset.StandardCharsets;
    import java.util.*;

    /**
     * A stream class supporting input and output semantics for any file-like
     * file system object (file or device).  The backing construct is not really
     * a "stream" because it must provide methods to get and set the current
     * offset (the current read/write position) into the stream.
     *
     * @author    GES
     */
    public class FileStream
    extends Stream
    {
       /** The maximum size of mapped memory "chunks". */
       protected static final int MAPPED_BUF_SZ = (256 * 1024);

       /** The size of the non-memory mapped write buffer. */
       private static final int WRITE_BUF_SZ = 4096;

       /**
        * Flag indicating if this is a memory-mapped stream.  Can be set only by sub-classes.
        */
       private final boolean memoryMapped;

       /** Flag to indicate if the last line in the file is duplicate (true for LOB files). */
       private final boolean hasDuplicatedLastLine;

       /** <code>true</code> if using memory mapped file support. */
       private boolean mapped = false;

       /** <code>true</code> if the file must be writable. */
       private boolean write = false;

       /** The backing construct for the "stream". */
       protected FileChannel file = null;

       /** Buffer for mapped file usage; this is in memory only if {@link #mapped} flag is {@code true}. */
       protected ByteBuffer binaryBuffer = null;

       /** The cached character decoder used in more complex operations. */
       protected CharsetDecoder decoder = null;

       /** The starting offset (in bytes) of the memory mapped buffer. */
       private long offset = 0;

       /** The size (in bytes) of the memory mapped buffer. */
       private long memsz = 0;

       /** Buffer used for non-memory-mapped writes. */
       private byte[] buffer = null;

       /** Number of bytes in the write buffer (non-memory-mapped mode). */
       private int pending = 0;

       /** The current byte mapping, if any, otherwise, {@code null}. */
       private Byte[] byteMapping = null;

       /** The name of the file. */
       protected final String fileName;

       /** The name of the directory of the file. */
       protected final String baseDirectoryName;

       /** The name of the directory to store LOB. */
       protected String lobDirectoryName;

       /** The current line nunber under reading for the source file. */
       private int currLineNumber = 0;

       /** Optional BOM signature size to be skipped for text files. */
       private int optionalBOMSize = 0;

       /**
        * Default constructor.  Inits some variables to default values.
        */
       public FileStream()
       {
          memoryMapped = false;
          fileName = null;
          baseDirectoryName = null;
          hasDuplicatedLastLine = true;
       }

       /**
        * Constructs an instance using a filename, this file or device will be
        * opened for reading or writing based on the given <code>write</code>
        * flag.  The current read/write position will be at offset 0 unless in
        * <code>write</code> mode and the <code>append</code> flag is on.
        *
        * @param    filename
        *           The name of the file or device to open.
        * @param    write
        *           If <code>true</code>, open in write mode, otherwise only
        *           open the file for reading.
        * @param    append
        *           In <code>write</code> mode, if <code>true</code>, the current
        *           file pointer is set to the end of the file. Otherwise if in
        *           <code>write</code> mode, the file is truncated to 0 length.
        *           This value is ignored in <code>read</code> mode.
        *
        * @throws   ErrorConditionException
        *           - If the file does not exist.
        *           - If the current user does not have the proper rights to open
        *             the file or device.
        */
       public FileStream(String filename, boolean write, boolean append)
       throws ErrorConditionException
       {
          this(filename, write, append, true, null);
       }

       /**
        * Constructs an instance using a filename, this file or device will be
        * opened for reading or writing based on the given <code>write</code>
        * flag.  The current read/write position will be at offset 0 unless in
        * <code>write</code> mode and the <code>append</code> flag is on.
        *
        * @param    filename
        *           The name of the file or device to open.
        * @param    write
        *           If <code>true</code>, open in write mode, otherwise only
        *           open the file for reading.
        * @param    append
        *           In <code>write</code> mode, if <code>true</code>, the current
        *           file pointer is set to the end of the file. Otherwise if in
        *           <code>write</code> mode, the file is truncated to 0 length.
        *           This value is ignored in <code>read</code> mode.
        * @param    memoryMapped
        *           Flag indicating if the mapping is done in memory.  Only for read-only files.
        * @param    fsr
        *           The file system resource to use for this stream. May be {@code null} if no resource
        *           checking is required.
        *
        * @throws   ErrorConditionException
        *           - If the file does not exist.
        *           - If the current user does not have the proper rights to open the file or device.
        */
       public FileStream(String filename,
                         boolean write,
                         boolean append,
                         boolean memoryMapped,
                         FileSystemResource fsr)
       throws ErrorConditionException
       {
          this(filename, write, append, memoryMapped, fsr, false);
       }

       /**
        * Constructs an instance using a filename, this file or device will be
        * opened for reading or writing based on the given <code>write</code>
        * flag.  The current read/write position will be at offset 0 unless in
        * <code>write</code> mode and the <code>append</code> flag is on.
        *
        * @param    filename
        *           The name of the file or device to open.
        * @param    write
        *           If <code>true</code>, open in write mode, otherwise only
        *           open the file for reading.
        * @param    append
        *           In <code>write</code> mode, if <code>true</code>, the current
        *           file pointer is set to the end of the file. Otherwise if in
        *           <code>write</code> mode, the file is truncated to 0 length.
        *           This value is ignored in <code>read</code> mode.
        * @param    memoryMapped
        *           Flag indicating if the mapping is done in memory.  Only for read-only files.
        * @param    fsr
        *           The file system resource to use for this stream. May be {@code null} if no resource
        *           checking is required.
        * @param    hasDuplicatedLastLine
        *           Flag to indicate if the last line in the file is duplicate (true for LOB files).
        *
        * @throws   ErrorConditionException
        *           - If the file does not exist.
        *           - If the current user does not have the proper rights to open the file or device.
        */
       public FileStream(String filename,
                         boolean write,
                         boolean append,
                         boolean memoryMapped,
                         FileSystemResource fsr,
                         boolean hasDuplicatedLastLine)
       throws ErrorConditionException
       {
          this.memoryMapped = memoryMapped;
          this.hasDuplicatedLastLine = hasDuplicatedLastLine;
          String fname = filename;
          String dname = null;
          if (write && !memoryMapped)
          {
             throw new IllegalArgumentException("Writable files must be mapped to memory!");
          }

          String errmsg = null;
          int    errnum = 0;

          try
          {
             File   target = new File(filename);
             String canonName = target.getCanonicalPath();
             if (fsr != null && (!fsr.canRead(canonName) || (write && !fsr.canWrite(canonName))))
             {
                throw new SecurityException();
             }

             String mode   = (write ? "rw" : "r");
             fname         = target.getName();
             dname         = target.getParent();

             // create a traditional file system resource with the proper mode
             @SuppressWarnings("resource")
             RandomAccessFile raf = new RandomAccessFile(target, mode);

             // obtain the backing channel (we no longer need access to the raf)
             file = raf.getChannel();

             // remember if the file is read-only or is read-write
             this.write = write;

             if (!write)
             {
                // read only files shift directly into memory mapped mode at the start of the file
                map(0);
                // compute optional BOM size to skip for text oriented reading operations
                if (!binary)
                {
                   BOM bomHeader = readBOM();
                   if (bomHeader != null)
                   {
                      optionalBOMSize = bomHeader.size;
                   }
                }
             }
             else
             {
                // writable files always start writing at the end of the file
                // which means that memory mapped support is not very useful
                // (since memory mapped file regions that are mapping parts of
                // the file that are past the end, cause the file to be extended
                // when the file should in fact not extend)
                mapped = false;

                // init the file pointer if as long as the file is not a device like /dev/null
                if (target.isFile())
                {
                   if (append)
                   {
                      file.position(file.size());
                   }
                   else
                   {
                      file.truncate(0);
                   }
                }
             }
          }
          catch (FileNotFoundException fnf)
          {
             if (dname != null && !new File(dname).isDirectory() && write)
             {
                errmsg = "" + dname + " is not a directory";
                errnum = 302;
             }
             else
             {
                errmsg = "\"" + filename + "\" was not found";
                errnum = 293;
             }
          }
          catch (SecurityException se)
          {
             errmsg = "You do not have permission to access " + filename;
             errnum = 294;
          }
          catch (IOException ioe)
          {
             // this error message is missing the " Errno=" portion that would
             // be there in Progress since there is no concept of this in Java
             // --> we also don't know if there are other cases in which this
             //     exception might map to a different error text and error num
             errmsg = "Unable to open file:" + filename + ".";
             errnum = 98;
          }

          if (errmsg != null)
          {
             ErrorManager.recordOrThrowError(errnum, errmsg);
          }
          fileName = fname;
          baseDirectoryName = dname != null ? dname : "";
       }

       /**
        * Flushes any data remaining in buffers to file or device.
        */
       @Override
       public void flush()
       {
          try
          {
             flushData();
             flushBuffered();
          }
          catch (IOException e)
          {
             // ignore exceptions
          }

          super.flush();
       }

       /**
        * The number of bytes available to be immediately read without blocking.
        *
        * @return   The number of available bytes.
        *
        * @throws  IOException
        *          if a problem was encountered wile processing the file.
        */
       @Override
       public long available()
       throws IOException
       {
          long num = 0;

          if (mapped)
          {
             num = binaryBuffer.remaining();

             if (num == 0)
             {
                // we must be at the end of the memory region, re-calc if there is anything left
                num = getLen() - (offset + memsz);
             }
          }
          else
          {
             // we must be writing at the end of the file because that is the
             // only non-mapped use case, but check this to confirm
             if (buffer == null)
             {
                // WARNING: this can be expensive
                num = file.size() - file.position();
             }
          }

          return num;
       }

       /**
        * The 0-based offset into the stream at which the next read or write will occur.
        *
        * @return  The current position in the stream.
        *
        * @throws  IOException
        *          if a problem was encountered wile processing the file.
        */
       @Override
       public long getPos()
       throws IOException
       {
          if (mapped)
          {
             return offset + binaryBuffer.position();
          }
          else
          {
             flushBuffered();
             return file.position();
          }
       }

       /**
        * Moves the current read/write position to the specified absolute 0-based offset.
        *
        * @param   pos
        *          The new read/write position in the stream.
        *
        * @throws  IOException
        *          if a problem was encountered wile processing the file.
        */
       @Override
       public void setPos(long pos)
       throws IOException
       {
          long sz = getLen();

          if (pos < sz)
          {
             if (mapped)
             {
                if (pos >= offset && pos < (offset + memsz))
                {
                   // the target location is inside the current memory region
                   binaryBuffer.position((int) (pos - offset)); // write ops
                }
                else
                {
                   // change our mapping
                   map(pos);
                }
             }
             else
             {
                // we can shift into mapped mode
                map(pos);
             }
          }
          else
          {
             if (mapped)
             {
                unmap();
             }

             // extend to the end or past the end of the file
             if (file != null)
             {
                file.position(pos);
             }
          }
       }

       /**
        * State of the input side of the stream.
        *
        * @return   <code>true</code> if the input side of the stream is active.
        */
       @Override
       public boolean isIn()
       {
          return (file != null);
       }

       /**
        * State of the output side of the stream.
        *
        * @return   <code>true</code> if the output side of the stream is active.
        */
       @Override
       public boolean isOut()
       {
          return (file != null);
       }

       /**
        * The length of the stream in bytes.
        *
        * @return  The length of the stream.
        *
        * @throws  IOException
        *          In the event of an error while getting the length of the stream.
        */
       @Override
       public long getLen()
       throws IOException
       {
          // this may change the file size
          flushBuffered();

          return file.size();
       }

       /**
        * Truncates or extends the stream to the specified length if this stream
        * supports such an operation. If truncation is requested, all data
        * located after this point in the file is discarded.  If extending the
        * file is requested, the values of the data in the extended portion of
        * the file is undefined.
        *
        * @param   len
        *          The new length of the file.
        *
        * @throws  IOException
        *          In the event of an error while setting the size of the stream.
        */
       @Override
       public void setLen(long len)
       throws IOException
       {
          long    sz    = getLen();
          long    pos   = getPos();
          boolean remap = false;

          if (len <= sz)
          {
             if (mapped)
             {
                if (len < (offset + memsz))
                {
                   // we are truncating the file in a way that affects the mem
                   // mapping
                   unmap();

                   remap = true;
                }

                // "true up" our current file position so the truncate can
                // manage the position properly
                file.position(pos);
             }

             // normal truncation
             file.truncate(len);
          }
          else
          {
             // file extend (Progress has no counterpart to this)
             file.position(len);

             if (mapped)
             {
                if (pos < len && pos + MAPPED_BUF_SZ > len)
                {
                   remap = true;
                }
             }
          }

          if (remap && pos < len)
          {
             // re-establish our mapping
             map(pos);
          }
       }

       /**
        * Write the given character to the output stream. On Windows non-binary streams, the
        * EOLN characters ({@code LF}) are converted to proper {@code CRLF} sequence.
        *
        * @param    ch
        *           The character to be written.
        *
        * @throws  IOException
        *          In the event of an error while writing the character.
        */
       @Override
       public void writeCh(char ch)
       throws IOException
       {
          if (ch == LF)
          {
             if (!binary && isWindows)
             {
                // on Windows OS, non [binary] the newline char \n (LF, 0x0A) are encoded using 2 chars:
                // \n\r (CRLF, 0x0D 0x0A). On Linux and binary Windows, the newline character is not altered
                write((byte) CR);
             }

             write((byte) LF);
             return;
          }

          // convert and write the encoded character (possible multi-byte)
          write(((convert && targetCharset != null) ? targetCharset : sourceCharset).encode(String.valueOf(ch))
                                                                                    .array());
       }

       /**
        * Write the given byte to the output stream. On Windows non-binary streams, the
        * EOLN characters ({@code LF}) are converted to proper {@code CRLF} sequence.
        *
        * @param   b
        *          The byte to be written.
        *
        * @throws  IOException
        *          In the event of an error while writing the byte.
        */
       @Override
       public void writeByte(byte b)
       throws IOException
       {
          // a byte is a byte, there should be no need for charset conversion
          write(b);
       }

       /**
        * Write the given string to the output stream.
        *
        * @param   data
        *          The data to be written.
        *
        * @throws  IOException
        *          In the event of an error while writing the string value.
        */
       @Override
       public void write(String data)
       throws IOException
       {
          if (!binary && isWindows)
          {
             // on Windows OS, non [binary] the newline char \n (LF, 0x0A) are encoded using 2 chars:
             // \n\r (CRLF, 0x0D 0x0A). On Linux and binary Windows, the newline character is not altered

             int len = data.length();
             char[] chars = new char[len * 2];
             int j = 0;

             for (int i = 0; i < len; i++, j++)
             {
                char c = data.charAt(i);
                if (c == LF)
                {
                   chars[j] = CR;
                   j++;
                }
                chars[j] = c;
             }

             // create a new array only if new characters were inserted in the loop
             if (len != j)
             {
                data = new String(chars, 0, j);
             }
          }

          write(data.getBytes((convert && targetCharset != null) ? targetCharset : sourceCharset));
       }

       /**
        * Peeks at the character from the current read position in the stream (reads a character from
        * the current read position in the stream without incrementing stream read position. The next
        * {@code peekCh()} and {@code readCh()} will return the same value).
        * <p>
        * The underlying stream subclass determines the content of the result. Byte oriented streams
        * such as pipes or files will return a byte while streams that generate keystrokes or
        * characters may return a DBCS or Unicode character.
        *
        * @return  The next character read from the stream, -1 on any failure and -2 upon an {@code EOF}.
        */
       @Override
       public int peekCh()
       {
          try
          {
             // skip BOM header for first data portion reading
             skipBOMHeader();

             int ch = readCharacterWorker(true);

             if (ch == -1)
             {
                // signal EOF
                return -2;
             }
             return ch;
          }
          catch (IOException ioe)
          {
             // non-EOF error thrown during read
             return -1;
          }
       }

       /**
        * Read the raw integer value of a character from the current read
        * position in the stream. This method DOES NOT convert the value read
        * from the stream's character set into Unicode.
        * <p>
        * At this time, only a single byte will be read no matter what encoding
        * is used by the underlying stream.  This means that DBCS or Unicode
        * encodings will not be properly handled.
        *
        * @return   The next character read from the stream, -1 on any failure
        *           and -2 upon an <code>EOF</code>.
        */
       @Override
       public int readCh()
       {
          try
          {
             // skip BOM header for first data portion reading
             skipBOMHeader();

             int ch = readCharacterWorker(false);

             if (ch == -1)
             {
                // signal EOF
                return -2;
             }

             return ch;
          }
          catch (IOException ioe)
          {
             // non-EOF error thrown during read
             return -1;
          }
       }

       /**
        * Read a single byte from the underlying stream.
        *
        * @return   A single byte from the stream.
        */
       @Override
       public int readByte()
       {
          try
          {
             int b = readByteWorker(false);

             if (b == -1)
             {
                // signal EOF
                return -2;
             }

             return b;
          }
          catch (IOException ioe)
          {
             // non-EOF error thrown during read
             return -1;
          }
       }

       /**
        * Read a chunk of bytes from the current read position in the stream to the
        * offset calculated by given length or to the <code>EOF</code>, whichever
        * comes first. The <code>EOF</code> character is not returned.
        *
        * @param    len
        *           The number of bytes to read.
        *
        * @return   The chunk read from the stream or <code>null</code> if there
        *           is nothing to read (the <code>EOF</code> has been reached). The
        *           length of the returned array will be the smaller of the
        *           <code>len</code> parameter or the actual bytes left before the
        *           <code>EOF</code>.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       @Override
       public byte[] readBytes(int len)
       throws IOException
       {
          byte[] chunk = new byte[len];

          int currChunkLen = 0;
          if (mapped && (currChunkLen = binaryBuffer.remaining()) == 0)
          {
             // try to get the next chunk (if it fails we must be at the end of the file and we will
             // shift into unmapped mode)
             map(offset + memsz);
          }

          // the position inside returning array
          int dstStart = 0;
          // total number of bytes copied
          int targetCopied = 0;
          while (mapped && (currChunkLen = binaryBuffer.remaining()) > 0)
          {
             // get next portion of data
             if (len - dstStart >= currChunkLen)
             {
                binaryBuffer.get(chunk, dstStart, currChunkLen);
                dstStart += currChunkLen;
                targetCopied += currChunkLen;
                map(offset + memsz);
             }
             else
             {
                // last or single copy operation
                binaryBuffer.get(chunk, dstStart, len - dstStart);
                targetCopied += len - dstStart;
                // need to stop reading here unconditionally
                break;
             }
          }

          // sometimes we need to remake the array to be returned or just return null if nothing was read
          return targetCopied == 0 ? null : targetCopied < len ? Arrays.copyOf(chunk, targetCopied) : chunk;
       }

       /**
        * Read all characters from the current read position in the stream to the next line separator (as
        * determined by the {@code File.separator} or to the {@code EOF}. Any line separator character(s) and the
        * {@code EOF} character are not returned.
        *
        * @return  The next line read from the stream.
        *
        * @throws  EOFException
        *          When attempting to read past the end of the file.
        * @throws  IOException
        *          In the event of an other error while reading the line.
        * @throws  InterruptedException
        *          If the thread was interrupted while it was blocked reading the stream.
        */
       @Override
       public String readLn()
       throws EOFException,
              IOException,
              InterruptedException
       {
          // skip BOM header for first data portion reading
          skipBOMHeader();

          // reading next line
          currLineNumber++;
          // read bytes until 'EOL' is found and after that decode them.
          ByteArrayOutputStream baos = new ByteArrayOutputStream();
          int bn = -1; // last byte read
          int bn_1 = -1; // second-to-last byte read
          boolean eolFound = false;
          while (!(eolFound = isNewLine(bn_1, bn)))
          {
             int b = readByteWorker(false);
             if (b == -1)
             {
                // EOF
                if (!hasDuplicatedLastLine)
                {
                   // end of file is considered end of line and the line is not ignored
                   eolFound = true;
                }
                break;
             }

             baos.write(b);

             if (bn != -1)
             {
                bn_1 = bn;
             }
             bn = b;
          }

          CharBuffer chars = baos.size() == 0 || (!eolFound && currLineNumber == 1)
                                 ? null : decode(baos.toByteArray());
          StringBuilder sb = new StringBuilder();
          if (chars != null)
          {
             sb.append(chars.toString());
          }

          // empty line to return for single line file without EOL marker
          return eolFound || currLineNumber > 1 ? readLineCleanup(sb) : "";
       }

       /**
        * Closes the input stream and releases OS resources associated with it.
        */
       @Override
       public void closeIn()
       {
          close();
       }

       /**
        * Closes the output stream and releases OS resources associated with it.
        */
       @Override
       public void closeOut()
       {
          close();
       }

       /**
        * Closes the stream and releases OS resources associated with it.
        */
       @Override
       public void close()
       {
          try
          {
             endOutput();
             cleanup(true, true);

             // force anything pending to be written to the file
             if (mapped)
             {
                unmap();
             }
             else
             {
                // sometimes we need to call pulse for empty frame (and if no pages rendered yet)
                // This is commented because headers/footers are unnecessary when the redirected output is
                // paginated and no output was explicitly sent to the file.
                // TODO Uncomment if a regression is detected.
                /* if (getPulseOnClose() && rawGetPageNum() == 1)
                {
                   pulseOutput();
                } */
                flushBuffered();

                // de-allocate our buffer in case our reference is kept around
                buffer = null;
             }

             if (file != null)
             {
                file.close();
                file = null;
             }
          }
          catch (IOException ioe)
          {
             // nothing to do
          }
       }

       /**
        * Assigns the internal stream reference to the given reference.
        *
        * @param   stream
        *          The new internal stream reference to use for all operations.
        *
        * @throws  UnsupportedOperationException
        *          Always since this type of stream objects does not support assign operations.
        */
       @Override
       public void assign(Stream stream)
       throws UnsupportedOperationException
       {
          throw new UnsupportedOperationException("Cannot handle this.");
       }

       /**
        * Write the given byte array to the output stream.
        *
        * @param    data
        *           The data to be written.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       @Override
       public void write(byte[] data)
       throws IOException
       {
          write(data, 0, data.length);
       }

       /**
        * Write the specified range of bytes from the given byte array to the output stream.
        *
        * @param    data
        *           The data to be written.
        * @param    off
        *           Starting offset in data from which to read bytes to be written. Must be
        *           non-negative and {@code &lt; data.length}.
        * @param    len
        *           Length of data to be written. Must be non-negative and {@code &lt;= (data.length
        *           - offset)}.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       @Override
       public void write(byte[] data, int off, int len)
       throws IOException
       {
          while (mapped && len > 0)
          {
             int num = binaryBuffer.remaining();

             if (num <= 0)
             {
                // no room at all, try to get the next chunk (if it fails we must
                // be at the end of the file and we will shift into unmapped mode)
                map(offset + memsz);
             }
             else
             {
                if (num >= len)
                {
                   // there is space and enough of it to output everything left
                   binaryBuffer.put(data, off, len);

                   // done
                   return;
                }
                else
                {
                   // partial space left
                   binaryBuffer.put(data, off, num);

                   off += num;
                   len   -= num;
                }
             }
          }

          // we may have done partial output above, but if we shifted out of
          // mapped mode or were not in mapped mode, then we must output now
          if (!mapped && len > 0)
          {
             // we must be writing to the end of the file
             writeWorker(data, off, len);
          }
       }

       /**
        * Write the entire block of data (as represented by the given <code>blob</code>) to the stream with no
        * delimiters, end of line insertions or other modifications.  The data will be exported into separate
        * file within subdirectory defined in LOB-DIR option or if not defined in current base directory.
        * <p>
        * Use of this method does NOT require bracketing between {@link #startOutput} and {@link #endOutput}
        * methods and these must not be called otherwise the result is undefined.
        * <p>
        * This method is called only from client-side, via the ExportField.output method.
        * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       @Override
       public void writeBlob(blob val)
       throws ErrorConditionException,
              StopConditionException
       {
          if (val.isUnknown())
          {
             // for unknown blob do not create separate file, just put ? (unquoted) to the master
             character unknown = new character();
             super.writeField(unknown);
          }
          else
          {
             // export blob data into separate file
             StringBuilder sb = new StringBuilder();
             sb.append(getLobDirectory());
             sb.append(File.separator);
             sb.append(val.getFieldName());
             sb.append(ExportField.getJavaPId());
             sb.append("_");
             sb.append(ExportField.getNextFSId());
             sb.append(".blb");
             String blobName = sb.toString();

             // write separate file and then close it
             // allow an empty file to be written, even if the blob has no contents.
             FileStream fsBlob = new FileStream(blobName, true, false);
             if (val.lengthOf() != 0)
             {
                fsBlob.writeBlock(val);
             }
             fsBlob.close();

             // export the reference to blob data into current stream
             super.writeField(blobName.substring(blobName.lastIndexOf(File.separator) + 1));
          }
       }

       /**
        * Write the entire block of data (as represented by the given <code>clob</code>) to the stream with no
        * delimiters, end of line insertions or other modifications.  The data will be exported into separate
        * file within subdirectory defined in LOB-DIR option or if not defined in current base directory.
        * <p>
        * Use of this method does NOT require bracketing between {@link #startOutput} and {@link #endOutput}
        * methods and these must not be called otherwise the result is undefined.
        * <p>
        * This method is called only from client-side, via the ExportField.output method.
        * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       @Override
       public void writeClob(clob val)
       throws ErrorConditionException,
              StopConditionException
       {
          if (val.isUnknown())
          {
             // for unknown clob do not create separate file, just put ? (unquoted) to the master
             character unknown = new character();
             super.writeField(unknown);
          }
          else
          {
             // export blob data into separate file
             StringBuilder sb = new StringBuilder();
             sb.append(getLobDirectory());
             sb.append(File.separator);
             sb.append(val.getFieldName());
             sb.append("!");
             // getting clob codepage or if not defined will use cpinternal
             String codePage = val._getCodePage();
             if (codePage == null)
             {
                codePage = I18nOps._getCPInternal();
             }
             sb.append(codePage );
             sb.append("!");
             sb.append(ExportField.getJavaPId());
             sb.append("_");
             sb.append(ExportField.getNextFSId());
             sb.append(".blb");
             String clobName = sb.toString();

             // write separate file and then close it
             // allow an empty file to be written, even if the clob has no contents.
             FileStream fsClob = new FileStream(clobName, true, false);
             if (val.lengthOf() > 0)
             {
                fsClob.startOutput(0);
                fsClob.writeField(val);
                fsClob.endOutput();
             }
             fsClob.close();

             // export the reference to blob data into current stream
             super.writeField(clobName.substring(clobName.lastIndexOf(File.separator) + 1));
          }
       }

       /**
        * Check if this is  a file resource.
        *
        * @return   Always <code>true</code>
        */
       @Override
       protected boolean isFileResource()
       {
          return true;
       }

       /**
        * Flushes this output stream and forces any buffered output bytes to be written out.
        * The general contract of flush is that calling it is an indication that,
        * if any bytes previously written have been buffered by the implementation of the output
        * stream, such bytes should immediately be written to their intended destination.
        *
        * @throws   IOException
        *           if an I/O error occurs.
        */
       @Override
       protected void flushData()
       throws IOException
       {
          if (file != null)
          {
             file.force(true);
          }
       }

       /**
        * Gets the stream's lob directory defined to put/get lob data fields.
        *
        * @return   The path for LOB directory, no trailing file separator.
        */
       private String getLobDirectory()
       {
          return lobDirectoryName != null ? lobDirectoryName :
                    baseDirectoryName == null || baseDirectoryName.isEmpty() ? "." : baseDirectoryName;
       }

       /**
        * Worker to output a single byte to our file at the current output
        * position.
        *
        * @param    b
        *           The data to write.
        */
       private void write(byte b)
       throws IOException
       {
          if (mapped && !binaryBuffer.hasRemaining())
          {
             // try to get the next chunk (if it fails we must be at the end of
             // the file and we will shift into unmapped mode)
             map(offset + memsz);
          }

          // if we are still mapped at this point, we must have room for a byte
          if (mapped)
          {
             binaryBuffer.put(b);
          }
          else
          {
             // we must be writing to the end of the file
             writeWorker(b);
          }
       }

       /**
        * Write a byte to the buffer, flushing if the buffer is full.
        *
        * @param    b
        *           The byte to write.
        */
       private void writeWorker(byte b)
       throws IOException
       {
          boolean written = false;

          if (pending < WRITE_BUF_SZ)
          {
             if (buffer == null)
                allocate();

             // there is room
             buffer[pending] = b;
             pending++;
             written = true;
          }

          if (pending == WRITE_BUF_SZ)
          {
             // the buffer is full
             flushBuffered();

             if (!written)
             {
                buffer[pending] = b;
                pending++;
             }
          }
       }

       /**
        * Worker that provides a mechanism to write a buffer directly to the
        * file channel at the current write position.
        *
        * @param    data
        *           The buffer to write.
        * @param    offset
        *           The starting index to output.
        * @param    len
        *           The number of bytes to output.
        */
       private void writeWorker(byte[] data, int offset, int len)
       throws IOException
       {
          // if the buffer will be filled up or overflowed by this new data OR if
          // there is nothing in the buffer but the new data is larger than or
          // equal to the buffer size, then output everything (buffered and new)
          // immediately
          if ((pending + len) >= WRITE_BUF_SZ)
          {
             if (pending > 0)
             {
                // write from 2 buffers in 1 call
                ByteBuffer[] bufs = new ByteBuffer[2];
                bufs[0] = ByteBuffer.wrap(buffer, 0, pending);
                bufs[1] = ByteBuffer.wrap(data, offset, len);

                int[] sz = new int[2];
                sz[0] = pending;
                sz[1] = len;

                writeFully(bufs, sz, (pending + len));
                pending = 0;
             }
             else
             {
                // nothing buffered, just make 1 simple call to write
                ByteBuffer buf = ByteBuffer.wrap(data, offset, len);
                writeFully(buf, len);
             }
          }
          else
          {
             // can fit in buffer
             allocate();

             int end = offset + len;

             // copy the data
             for (int i = offset; i < end; i++)
             {
                buffer[pending] = data[i];
                pending++;
             }
          }
          afterWrite();
       }

       /**
        * Read a character from the file at the current file pointer.
        *
        * @param    peek
        *           {@code true} if the character is only peeked, ie the file pointer is not incremented.
        *
        * @return   The next character read or -1 if EOF is reached.
        */
       private int readCharacterWorker(boolean peek)
       throws IOException
       {
          long currentPos = -1;
          if (peek)
          {
             // save current position
             currentPos = getPos();
          }

          try
          {
             ByteArrayOutputStream baos = new ByteArrayOutputStream();
             while (true)
             {
                int b = readByteWorker(false);
                if (b == -1)
                {
                   // EOF reached
                   break;
                }

                baos.write(b);

                // decode them
                byte[] bytes = baos.toByteArray();
                CharBuffer ch = decode(bytes, false);

                if (ch != null)
                {
                   char c = ch.get();

                   if (ch.remaining() > 0)
                   {
                      throw new IOException("More than one character was read from the stream!");
                   }

                   return c;
                }
             }

             if (baos.size() == 0)
             {
                // EOF
                return -1;
             }

             // otherwise, we could not decode the bytes...
             throw new IOException("EOF reached - could not decode the bytes from the stream!");
          }
          finally
          {
             if (peek)
             {
                setPos(currentPos);
             }
          }
       }

       /**
        * Read a byte from the file at the current file pointer.
        *
        * @param   peek
        *          {@code true} if the byte is only peeked, ie the file pointer is not incremented.
        *
        * @return   The next byte read or -1 if EOF is reached.
        */
       private int readByteWorker(boolean peek)
       throws IOException
       {
          int b = -1;

          if (mapped && !binaryBuffer.hasRemaining())
          {
             // try to get the next chunk (if it fails we must be at the end of the file and we will
             // shift into unmapped mode)
             map(offset + memsz);
          }

          // unmapped mode only occurs at the end of the file, so if we are not mapped, then there is
          // nothing to read, protect from this case
          if (mapped && binaryBuffer.hasRemaining())
          {
             // there must be at least 1 character available to read (it must be widened without sign extension)
             if (peek)
             {
                binaryBuffer.mark();
             }
             // binaryBuffer returns signed bytes, make sure to make them unsigned not to confuse -1 with EOF
             b = (binaryBuffer.get() & 0x000000FF);
             if (peek)
             {
                binaryBuffer.reset();
             }
          }

          return b;
       }

       /**
        * Allocate the write buffer if it is not already allocated.
        */
       private void allocate()
       {
          if (buffer == null)
          {
             buffer  = new byte[WRITE_BUF_SZ];
             pending = 0;
          }
       }

       /**
        * Output any buffered writes to the file. This only affects writes at
        * the end of the file (non-mapped mode). If called while in mapped mode
        * this is a NOP.
        */
       private void flushBuffered()
       throws IOException
       {
          if (!mapped && buffer != null && pending > 0)
          {
             ByteBuffer buf = ByteBuffer.wrap(buffer, 0, pending);
             writeFully(buf, pending);
             pending = 0;
          }
       }

       /**
        * Write <code>len</code> bytes from the given buffer to the file before
        * returning.
        *
        * @param    buf
        *           The buffer to output.
        * @param    len
        *           The size of the buffer in bytes.  This MUST NOT be different
        *           than the amount of data in the buffer.  It is provided here
        *           only for efficiency since the callers of this method already
        *           have this value.
        */
       private void writeFully(ByteBuffer buf, int len)
       throws IOException
       {
          int num = 0;

          while (num < len)
          {
             int next = file.write(buf);

             num += next;

             if (next > 0 && num < len)
             {
                // there must be more in the buffer
                buf.compact();
             }
          }
          afterWrite();
       }

       /**
        * Write <code>len</code> bytes from the given buffer to the file before
        * returning. This takes advantage of the NIO scatter/gather functionality
        * to improve performance.
        *
        * @param    buf
        *           The list of buffers to output.
        * @param    sz
        *           The size in bytes of each corresponding buffer. The length of
        *           this array must be the same as that of <code>buf</code>.
        * @param    len
        *           The cumulative size of the data in all buffers in bytes.  This
        *           MUST NOT be different than the amount of data in the buffer.
        *           It is provided here only for efficiency since the callers of
        *           this method already have this value.
        */
       private void writeFully(ByteBuffer[] buf, int[] sz, int len)
       throws IOException
       {
          int num    = 0;
          int start  = 0;
          int amount = buf.length;

          while (num < len)
          {
             int next = (int) file.write(buf, start, amount);

             num += next;

             if (next > 0 && num < len)
             {
                int remain = next;

                // there must be more in the buffer, find the place in the list
                // of buffers where we need to restart writing
                for (int i = start; i < amount && remain > 0; i++)
                {
                   if (sz[i] <= remain)
                   {
                      // this entry was fully output, bypass it on the next write
                      start++;
                      amount--;
                      remain -= sz[i];
                   }
                   else
                   {
                      // this entry was partially written, compact it
                      sz[i] -= remain;
                      buf[i].compact();
                      remain = 0;
                   }
                }
             }
          }
          afterWrite();
       }

       /**
        * Attempt to detect the file encoding based on the BOM.
        *
        * @return  A {@code BOM} structure containing data about character encoding if one is detected,
        *          otherwise {@code null}.
        */
       protected BOM readBOM()
       {
          MappedByteBuffer map;
          try
          {
             map = file.map(FileChannel.MapMode.READ_ONLY, 0L, Math.min(4, file.size()));
          }
          catch (IOException e)
          {
             return null;
          }

          byte[] magic;
          if (map.hasArray())
          {
             magic = map.array();
          }
          else
          {
             int n = map.remaining();
             magic = new byte[n];
             for (int i = 0; i < magic.length; i++)
             {
                magic[i] = map.get();
             }
          }

          return detectBOM(magic);
       }

       /**
        * Attempt to detect the encoding based on the BOM.
        *
        * @return  A {@code BOM} structure containing data about character encoding if one is detected,
        *          otherwise {@code null}.
        */
       public static BOM detectBOM(byte[] magic)
       {
          // Check the UTF-32BE BOM: 0x00_00_FE_FF
          if (magic.length >= 4        &&
             (0xFF & magic[0]) == 0x00 &&
             (0xFF & magic[1]) == 0x00 &&
             (0xFF & magic[2]) == 0xFE &&
             (0xFF & magic[3]) == 0xFF)
          {
             return BOM.UTF32;
          }
          // Check the UTF-32LE BOM: 0xFF_FE_00_00
          else if (magic.length >= 4         &&
             (0xFF & magic[0]) == 0xFF &&
             (0xFF & magic[1]) == 0xFE &&
             (0xFF & magic[2]) == 0x00 &&
             (0xFF & magic[3]) == 0x00)
          {
             return BOM.UTF32LE;
          }
          // Check the UTF-16BE BOM: 0xFE_FF
          else if (magic.length >= 2         &&
             (0xFF & magic[0]) == 0xFE &&
             (0xFF & magic[1]) == 0xFF)
          {
             return BOM.UTF16;
          }
          // Check the UTF-16LE BOM: 0xFF_FE
          else if (magic.length >= 2         &&
             (0xFF & magic[0]) == 0xFF &&
             (0xFF & magic[1]) == 0xFE)
          {
             return BOM.UTF16LE;
          }
          // Check the UTF-8 BOM: 0xEF_BB_BF
          else if (magic.length >= 3         &&
             (0xFF & magic[0]) == 0xEF &&
             (0xFF & magic[1]) == 0xBB &&
             (0xFF & magic[2]) == 0xBF)
          {
             return BOM.UTF8;
          }
          return null;
       }

       /**
        * Sets the stream's source codepage and activates character conversion mode.
        *
        * @param cp The source codepage for character conversion mode.
        */
       @Override
       public void setConvertSource(String cp)
       {
          super.setConvertSource(cp);

          if (sourceCharset != null)
          {
             decoder = sourceCharset.newDecoder();
             decoder.onMalformedInput(CodingErrorAction.REPORT);
             decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
          }
          else
          {
             decoder = null;
          }
       }

       /**
        * Sets the stream's lob directory located as a subdirectory for opened current file stream directory.
        *
        * @param    lobDir
        *           The name for subdirectory to store LOB files.
        */
       @Override
       public void setLobDirectory(String lobDir)
       {
          // simple check to avoid NPE
          if (lobDir != null && !lobDir.isEmpty() && baseDirectoryName != null)
          {
             StringBuilder sb = new StringBuilder();
             // if base directory is specified - add file separator, otherwise current directory is assumed
             if (!baseDirectoryName.isEmpty())
             {
                sb.append(baseDirectoryName);
                // add the file separator only if necessary
                if (!baseDirectoryName.endsWith(File.separator))
                {
                   sb.append(File.separator);
                }
             }
             // add lob subdir without path separator
             for (int i = 0; i < lobDir.length(); i++)
             {
                char chCurr = lobDir.charAt(i);
                if (chCurr != '\\' && chCurr != '/')
                {
                   sb.append(lobDir.substring(i));
                   break;
                }
             }
             // need to reconstruct LOB directory based on current file name
             lobDirectoryName = sb.toString();
          }
       }

       /**
        * Sets a new mapping byte array. Note that the method DOES NOT make a copy of the array, but it is
        * guaranteed that this object does not alter the array in any way so that the same instance of the array
        * may be used by multiple {@code FileStream}s at no memory/CPU expense to copy and store individual
        * instances of the array.
        *
        * @param   newByteMapping
        *          The new byte mapping. An array of 256 bytes values with mapping of bytes. All bytes must be
        *          mapped. If no special mapping is necessary use {@code newByteMapping[i] = i;}. Use {@code null}
        *          to disable byte mapping.
        */
       public void setInvalidBytesMapping(Byte[] newByteMapping)
       {
          if (newByteMapping != null && newByteMapping.length != 256)
          {
             throw new IllegalArgumentException("Byte mapping array must be 256 bytes long!");
          }

          byteMapping = newByteMapping;
       }

       /**
        * Sets the current read and/or write position in the stream, if the
        * stream type supports the operation.
        * <p>
        * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They
        * are mapped to the underlying Java platform that can handle streams with offsets that are
        * based on a 64-bit <code>long</code>.
        * <p>
        * A negative offset is ignored (no action is taken).
        * <p>
        * If the specified offset is larger than the length of the stream, in
        * Progress the seek position will report as the number requested, but if
        * no subsequent writes occur to the file, the file size is left the same
        * as it was before setting the position.  If writes do occur, all writes
        * occur at the new position. In this case, the resulting file will be
        * sized based on the requested offset + the number of bytes written.
        *
        * @param    newpos
        *           The new offset.  If this is the <code>unknown value</code>,
        *           the set operation will silently return without any change.
        */
       @Override
       public void setPosition(long newpos)
       {
          // explicit usage of the seek statement eliminates optional BOM skip
          if (optionalBOMSize > 0 && newpos >= 0 && newpos < optionalBOMSize)
          {
             optionalBOMSize = 0;
          }

          super.setPosition(newpos);
       }

       /**
        * Obtains the current read and/or write position in the stream, if the stream type supports
        * such a query.
        * <p>
        * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They
        * are mapped to the underlying Java platform that can handle streams with offsets that are
        * based on a 64-bit <code>long</code>.
        *
        * @return  The current offset into the stream or the <code>unknown value</code> if the
        *          stream type doesn't support a stream position.
        */
       @Override
       public int64 getPosition()
       {
          try
          {
             // skip BOM header for first data portion reading
             skipBOMHeader();
          }
          catch (IOException ioe)
          {
             // let the superclass to care of possible errors
          }

          return super.getPosition();
       }

       /**
        * Setup memory mapping starting at the given offset and extending to the end of the file or to the maximum
        * chunk size, whichever is smaller. If the file size is 0 or the starting offset is greater than or equal
        * to the file size, then shift into unmapped mode.
        *
        * @param    start
        *           The starting offset (0-based) in bytes.
        */
       protected void map(long start)
       throws IOException
       {
          // flush if this is a re-mapping
          if (mapped)
          {
             if (binaryBuffer != null && memoryMapped)
             {
                ((MappedByteBuffer) binaryBuffer).force();
             }
          }
          else
          {
             flushBuffered();
          }

          // this must follow any flushing since the file size can change as as result of flushing buffered writes
          long fileSize = getLen();
          long sz = fileSize;

          if (sz == 0 || start >= sz)
          {
             if (mapped)
             {
                unmap();
             }

             return;
          }

          mapped = true;
          sz     = Math.min((sz - start), MAPPED_BUF_SZ);
          offset = start;// remember the starting offset and size of the mapped buffer
          memsz  = sz;

          initBinaryBuffer(start, sz);
       }

       /**
        * Initialize the binary buffer storage with content of the file or steam to be read depending on class
        * implementation..
        *
        * @param    start
        *           The starting position inside source buffer.
        * @param    size
        *           The size of the binary buffer to re-allocate.
        *
        * @throws   IOException
        *           In the event of an error when decoding the bytes.
        */
       protected void initBinaryBuffer(long start, long size)
       throws IOException
       {
          if (memoryMapped)
          {
             FileChannel.MapMode mode = write ? FileChannel.MapMode.READ_WRITE : FileChannel.MapMode.READ_ONLY;
             binaryBuffer = file.map(mode, start, size);
          }
          else
          {
             binaryBuffer = ByteBuffer.allocate((int) size);
             file.read(binaryBuffer, start);
             binaryBuffer.rewind();
          }
       }

       /**
        * Decode the specified bytes using the {@link Stream#sourceCharset} or the cached {@link #decoder}, if set.
        * <p>
        * For multi-byte characters, this API expects for all of them to be available in the given array.
        *
        * @param    bytes
        *           The bytes to be decoded.
        *
        * @return   The decoded characters.
        *
        * @throws   IOException
        *           In the event of an error when decoding the bytes.
        */
       protected CharBuffer decode(byte[] bytes)
       throws IOException
       {
          return decode(bytes, true);
       }

       /**
        * Decode the specified bytes using the {@link Stream#sourceCharset} or the cached {@link #decoder}, if set.
        * <p>
        * For multi-byte characters, this API expects for all of them to be available in the given array.
        *
        * @param    bytes
        *           The bytes to be decoded.
        * @param    throwIO
        *           Flag indication if {@link IOException} will be throw in case of decoding issues.
        *
        * @return   The decoded characters.
        *
        * @throws   IOException
        *           In the event of an error when decoding the bytes.
        */
       protected CharBuffer decode(byte[] bytes, boolean throwIO)
       throws IOException
       {
          CharBuffer charBuffer = null;

          ByteBuffer buffer = ByteBuffer.wrap(bytes);

          // convert multibyte binary to character data using the charset
          if (decoder == null)
          {
             // if a decoder was not installed, use the static method
             charBuffer = sourceCharset.decode(buffer);
             return charBuffer;
          }

          boolean readError = false;
          while (buffer.remaining() != 0)
          {
             try
             {
                CharBuffer buf = decoder.decode(buffer);
                if (charBuffer == null)
                {
                   charBuffer = buf;
                }
                else
                {
                   charBuffer.append(buf);
                }
             }
             catch (CharacterCodingException cce)
             {
                long errOffset = getPos() - buffer.remaining();
                byte errByte = buffer.get();

                if (!mappingInvalidBytesEnabled())
                {
                   if (throwIO)
                   {
                      throw new IOException(String.format("%s: Invalid byte %#02x at offset %#010x%n",
                                                          fileName, errByte, errOffset),
                                            cce);
                   }
                   else
                   {
                      return null;
                   }
                }

                readError = true;
                Byte mapped = byteMapping[errByte & 0xFF];

                // try to fix
                int crtPos = buffer.position();
                if (mapped != null)
                {
                   // replace the invalid byte
                   buffer.put(crtPos - 1, mapped);
                   System.out.format("%s: Invalid byte %#02x at offset %#010x replaced with %#02x%n",
                                     fileName, errByte, errOffset, mapped);
                }
                else
                {
                   // drop the invalid byte
                   ByteBuffer buffer2 = ByteBuffer.allocate(buffer.capacity() - 1);
                   buffer2.put(buffer.array(), 0, crtPos - 1);
                   buffer2.put(buffer.array(), crtPos, buffer.remaining());
                   buffer2.position(crtPos - 1);
                   buffer = buffer2;
                   System.out.format("%s: Invalid byte %#02x at offset %#010x was dropped.%n",
                                     fileName, errByte, errOffset);
                }
             }
          }

          // give it another try in case bytes were mapped
          if (readError)
          {
             buffer.rewind();

             try
             {
                charBuffer = decoder.decode(buffer);
             }
             catch (CharacterCodingException cce)
             {
                long errOffset = getPos() - buffer.remaining();
                byte errByte = buffer.get();

                if (throwIO)
                {
                   throw new IOException(String.format("%s: Invalid byte %#02x at offset %#010x after maping%n",
                                                       fileName, errByte, errOffset),
                                         cce);
                }
                else
                {
                   return null;
                }
             }
          }

          return charBuffer;
       }

       /** Test whether a bye mapping array is in use. */
       private boolean mappingInvalidBytesEnabled()
       {
          return byteMapping != null;
       }

       /**
        * Skips file internal pointer for the text files to the value of the BOM size from the file beginning.
        * Not all files are considering as text ones and not all text files have BOM signature so this is
        * optional.
        */
       private void skipBOMHeader()
       throws IOException
       {
          if (getPos() == 0 && optionalBOMSize > 0)
          {
             setPos(optionalBOMSize);
          }
       }

       /**
        * Clear/release the memory mapped buffer.
        */
       protected void unmap()
       {
          // flush before clearing state
          if (binaryBuffer != null && memoryMapped)
          {
             ((MappedByteBuffer) binaryBuffer).force();
          }

          mapped = false;
          offset = 0;
          memsz  = 0;
          binaryBuffer = null;
       }

       /** A structure that identifies the know types of BOMs and their useful properties. */
       public enum BOM
       {
          UTF8("UTF-8", 3, 1, StandardCharsets.UTF_8),
          UTF16("UTF-16", 2, 2, StandardCharsets.UTF_16),
          UTF16BE("UTF-16BE", 2, 2, StandardCharsets.UTF_16BE),
          UTF16LE("UTF-16LE", 2, 2, StandardCharsets.UTF_16LE),
          UTF32("UTF-32", 4, 4, null),
          UTF32BE("UTF-32BE", 4, 4, null),
          UTF32LE("UTF-32LE", 4, 4, null);

          /** The BOM's official name. */
          public final String name;

          /** The size of BOM, in bytes. These bytes must be skipped at the start of a file. */
          public final int size;

          /** The expected bytes per character for ASCII. Useful for counting bytes in PSC footer. */
          public final int bpc;

          /** The Java implementation of the {@code Charset} handling the BOM's encoding. */
          public final Charset javaCs;

          /**
           * The private constructor. Initializes the immutable object's internal data.
           *
           * @param   name
           *          the BOM's name. Useful for debugging, too.
           * @param   size
           *          the size of BOM, in bytes. These bytes must be skipped at the start of a file.
           * @param   bpc
           *          the expected bytes per character for ASCII. Useful for counting bytes in PSC footer.
           * @param   javaCs
           *          the Java implementation of the {@code Charset} handling the BOM's encoding.
           */
          BOM(String name, int size, int bpc, Charset javaCs)
          {
             this.name = name;
             this.size = size;
             this.bpc = bpc;
             this.javaCs = javaCs;
          }
       }
    }