# Stream.java

    /*
    ** Module   : Stream.java
    ** Abstract : represents a Progress compatible stream for reading/writing
    **
    ** Copyright (c) 2005-2025, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- ----------------------------------Description-----------------------------------
    ** 001 GES 20050712   @21742 Created initial version supporting methods
    **                           for the standardized open/close, and both
    **                           formatted and unformatted reading/writing
    **                           of streams. Also has facilities for seek,
    **                           line and page processing.
    ** 002 GES 20050801   @21941 New assign() interface for support of shared streams.
    ** 003 GES 20050824   @22214 Implemented error processing including
    **                           silent error mode. Implemented the
    **                           Finalizable interface to provide for
    **                           automatic cleanup.
    ** 004 GES 20050907   @22593 Added support for the memptr type (see
    **                           readBlock() and writeBlock()).
    ** 005 ECF 20051104   @23236 Added no-op iterate() method. Required to
    **                           satisfy expanded Finalizable interface.
    ** 006 GES 20060115   @23921 Changed readLine to support BDT not just
    **                           character.
    ** 007 GES 20060207   @24328 Translate InterruptedException into a
    **                           StopConditionException.
    ** 008 GES 20060208   @24393 Added echo support and cleaned up docs.
    ** 009 GES 20060217   @24669 Added writeByte().
    ** 010 GES 20060225   @24743 Added footer and header support.
    ** 011 GES 20060226   @24752 Added methods to remote the paging state.
    ** 012 GES 20060227   @24772 Final footer/header support.
    ** 013 GES 20060302   @24857 Minor changes to enable removal from the
    **                           finalizable lists.
    ** 014 ECF 20060330   @25261 Throw EOFException from readNextLine() when
    **                           no next line is available.
    ** 015 GES 20060412   @25489 Added putField() signatures.
    ** 016 GES 20060517   @26194 Match new Finalizable interface requirement.
    ** 017 GES 20060523   @26455 Fix for skipping multiple sequential
    **                           delimiters in readFieldWorker() instead of
    **                           returning an empty string.
    ** 018 GES 20060817   @28662 Fix to ensure that all paged streams get a
    **                           form feed at the end.  Previously this was
    **                           only done in the cleanup processing for
    **                           streams with footers.
    ** 019 GES 20060905   @29226 Fixes for embedded null byte processing in
    **                           field reading.
    ** 020 GES 20060925   @29905 Added high level equivalents to the PUT and
    **                           EXPORT language statements instead of
    **                           requiring the caller to implement the start
    **                           and end buffering and the individual field
    **                           output.
    ** 021 GES 20061004   @30135 Added support for bracketing PUT in nested
    **                           error mode enable/disable processing.  This
    **                           duplicates a quirk in PUT processing where
    **                           a nested error will be honored after the
    **                           original error has been processed (so it is
    **                           as if there are 2 errors rather than 1 as
    **                           would normally be the case).
    ** 022 GES 20061102   @30906 Added hasPageHeadersFooters() query method.
    ** 023 GES 20061113   @31101 Modified the redirected terminal output
    **                           processing to always group buffered output
    **                           together (it now keeps all this output on
    **                           the same page, even though that may cause the
    **                           resulting page size to be smaller or larger
    **                           than specified for the stream).
    ** 024 GES 20061115   @31169 Added a page break notification framework.
    ** 025 NVS 20070516   @33545 Page headers and footers may do registrations
    **                           multiple times. Fixed the headers and footers
    **                           cumulative size calculation to allow it.
    ** 026 NVS 20070601   @33918 Page break listeners are removed from the
    **                           registry after they process notification.
    ** 027 NVS 20070611   @34008 Added pulseOutput() method and protected
    **                           processHeaders() method from recursion.
    ** 028 NVS 20070716   @34542 Fixed putUnformatted() method which should
    **                           output NL. Added registerCLoseListener()
    **                           method and stream close notification for all
    **                           registered listeners.
    ** 029 NVS 20070716   @34561 #028 was insufficient to solve the new line
    **                           issue with putUnformatted(), so I've added
    **                           variations of methods that take a flag that
    **                           controls the auto-append of new line.
    ** 030 NVS 20070724   @34678 Fixed close notification processing (doneOut)
    **                           and added buffer flushing after headers and
    **                           footers.
    ** 031 NVS 20070725   @34683 Added isIn() and isOut() abstract methods.
    ** 032 NVS 20070730   @34737 Redirected mode flushes terminal's buffers
    **                           after every page element.
    ** 033 NVS 20070731   @34755 Added registerAdvanceListener() method. Used
    **                           to notify all registered listeners about
    **                           page advance being performed.
    ** 034 NVS 20070802   @34767 Fixed advancePage() method which should check
    **                           the page line number after sending page
    **                           advance notification.
    ** 035 NVS 20070802   @34776 Added registerFlushListener() and flush()
    **                           methods. They provide a way to notify all
    **                           registered listeners about flushing being
    **                           performed on the stream.
    ** 036 NVS 20070828   @34923 Removed some debugging leftovers.
    ** 037 SIY 20070829   @34924 Unknown date should be represented as spaces
    **                           in output (Ref: @34927).
    ** 038 NVS 20070906   @34972 Since now page elements rendering can be done
    **                           while switching streams for a frame,
    **                           processRendering() makes sure a proper
    **                           redirection is established for the duration
    **                           of this rendering.
    ** 039 SIY 20070914   @35075 Fixed handling of some cases of line feeds.
    **                           Ref: #35073
    ** 040 SIY 20070924   @35197 Deeper rework of line feeds handling.
    **                           Ref: #35198
    ** 041 SIY 20070925   @35202 Fixed regression caused by #39: EXPORT
    **                           immediately after PUT inserts redundant
    **                           delimiter before first field.
    **                           Ref: #35199#1, #35201.
    ** 042 SIY 20070928   @35326 Fix for PUT/DISPLAY handling regressions.
    **                           Ref: #35314
    ** 043 EVL 20071005   @35374 Removing finalize() method. We can not control
    **                           when it is called.
    ** 044 EVL 20071008   @35396 Fixed redundant NL's.
    ** 045 EVL 20071010   @35410 Added the method to check the validity of
    **                           the stream.
    ** 046 NVS 20071018   @35483 Fixed a bug with delayed flushing of the
    **                           current output buffer putting Form Feeds far
    **                           away from the true page break point.
    ** 047 SIY 20071018   @35490 Fixes in NL handling in control mode.
    **                           Ref: #35486
    ** 048 SIY 20071023   @35498 Fixes regression caused by change in NL
    **                           handling (#047).
    ** 049 NVS 20071025   @35520 An attempt to solve the problem of split
    **                           output due to page overflow. Now
    **                           flushControlled() redraws page headers
    **                           internally and merges the recursively created
    **                           output in the proper order.
    ** 050 SIY 20071028   @35609 Fixed NPE.
    **                           Ref: #35579.
    ** 051 SIY 20071030   @35629 Fixed other regression caused by (#047).
    **                           Page break should be handled only when
    **                           NL is inserted. Also, changed default trailing
    **                           NL flag to true.
    ** 052 SIY 20071106   @35714 Minor change in javadoc.
    ** 053 SIY 20071116   @35823 Do not include trailing empty line when
    **                           checking for page break.
    **                           Ref: #35800.
    ** 054 CA  20071127   @36033 Added field newLineRequired used to output a
    **                           NL just before the page break. Changed access
    **                           modifier for method processHeaders from
    **                           private to public. Ref: #36031
    ** 055 NVS 20071212   @36321 Introduced the frozen list of page headers.
    **                           Registrations and deregistrations are done
    **                           against a live list, which gets frozen by
    **                           means of producing a snapshot of the list.
    **                           It's the frozen list that supplies headers
    **                           info for the stream when processing page
    **                           breaks. Freezing happens with the very first
    **                           output operation by calling a newly added
    **                           method freezeHeaders(). Once processed, the
    **                           frozen list gets destroyed and should be
    **                           recreated for the new page similarly.
    ** 056 GES 20071218   @36429 Removed unnecessary member.
    ** 057 CA  20071219   @36543 The skip option inside a put statement is
    **                           executed on the same page and will check for
    **                           page-break before and after it is executed.
    **                           The cleanUp method must take into account if
    **                           a put statement output some data on the first
    **                           line of a page, without advancing the cursor
    **                           to the next line (this is needed to output
    **                           correct the footers and the page-break).
    **                           Fixed page-headers/footers generation when
    **                           put statements are used. Ref: 36542
    ** 058 NVS 20071229   @36619 Introduced contiguous output mode and means
    **                           to control it. Made flushControlled() a void
    **                           method. Fixed page headers with advancePage
    **                           method (PAGE statement).
    **                           Added getAvailableLines(), isFreshPage() and
    **                           forcePageBreak() methods.
    ** 059 GES 20080103   @36660 Back out H055. That change was meant to solve
    **                           an issue where there was a registered page
    **                           element and pending output in the redirected
    **                           terminal buffer and at the time of flushing
    **                           that buffer, the page element had been
    **                           removed so the output from that element was
    **                           never seen.  In the 4GL the output was seen.
    **                           The essential problem is that the page
    **                           elements need to be forced to output at the
    **                           moment the first output is made for a page.
    **                           We were deferring that output too long. The
    **                           H055 attempted to solve this problem by
    **                           remembering the page element list at the
    **                           moment of first output.  But this change
    **                           causes a great deal of breakage in other
    **                           use cases AND it doesn't solve all possible
    **                           issues. The correct solution has the
    **                           redirected terminal call processHeaders()
    **                           on the associated stream when it is adding
    **                           output for the first time to a fresh buffer.
    ** 060 CA  20080103   @36655 If the buffer size is p = PAGE-SIZE -
    **                           LINE-COUNTER + 1, then flushControlled() must
    **                           write the buffer content on the same page, as
    **                           there are p lines available (the line on row
    **                           LINE-COUNTER is included too in the output).
    **                           Ref: #36540
    ** 061 CA  20080107   @36699 Back out the H059 change in extendPage() -
    **                           is OK to start the counter from LINE-COUNTER.
    ** 062 GES 20080104   @36702 advancePage now supports extendPage when
    **                           page footers are in use.
    ** 063 CA  20080108   @36723 endOutput() should increment the LINE-COUNTER
    **                           if a NL is appended. Ref: #36706
    **                           flushControlled() needs to compare the
    **                           remaining lines against the actual buffer
    **                           size. Ref: #36705
    ** 064 CA  20080115   @36816 Added method processFooters() which will
    **                           process the footers in a LIFO order.
    ** 065 GES 20080108   @36723 Changes to make flushing more correct. In
    **                           particular, many cases of inappropriate
    **                           recursion in page header/footer processing
    **                           were eliminated. It should never be possible
    **                           for header processing to trigger footer
    **                           processing or vice versa. Exposed some state
    **                           and new controls regarding paging and
    **                           flushing.
    ** 066 CA  20080118   @36850 Fixed page-break output when flushing a
    **                           buffer with size 2 and ending with a NL and
    **                           there are not enough rows in the current page
    **                           for the buffer content.
    ** 067 SIY 20080125   @36913 Fixed PUT SKIP(n) with n != 0 for streams
    **                           without explicitly set page size.
    **                           Ref: #36912
    ** 068 CA  20080303   @37233 Added 'unnamed' field, which is true if this
    **                           is an unnamed stream. Ref: #37225
    ** 069 EVL 20080312   @37433 Changed the approach of calculation pageNum.
    **                           This fixes the difference between 4gl and p2j
    **                           when the real report output is empty.
    ** 070 SIY 20080317   @37455 Added support for terminal stream.
    **                           Ref: #37453
    ** 071 GES 20080403   @37826 Added interface for readField(BDT[]).
    ** 072 GES 20080412   @38021 Added partial support for the convert option.
    **                           Fixed readFieldWorker() to handle embedded
    **                           newlines in strings and to use a more
    **                           appropriate conversion from export format
    **                           (the previous approach used a worker that
    **                           did more than just stripping the enclosing
    **                           quotes and replacing the doubled double
    **                           quotes with single double quotes).
    ** 073 GES 20080418   @38046 Added raw support to readField(). Fixed
    **                           edge cases for readFieldWorker() to allow
    **                           input buffer extension when a quote char is
    **                           just before the embedded newline (either a
    **                           start of string quote char or the 2nd in a
    **                           doubled up pair of escaped quotes).
    ** 074 SIY 20080418   @38033 Do not throw an error condition if some
    **                           amount of data was read successfully.
    **                           Ref: #38032
    ** 075 GES 20080425   @38123 Refactored to eliminate round trips to the
    **                           client by passing more data in a single
    **                           invocation. This improves performance.
    ** 076 SIY 20080515   @38285 Extend page when page break is forced by
    **                           view() or down().
    ** 077 SIY 20080515   @38286 Improved detection of presence output on the
    **                           page. Since page break resets trailingNL flag
    **                           to initial value (true), then any output
    **                           without NL at the page start will set it to
    **                           false. Any output with NL will increase line
    **                           counter, so presence of output will still be
    **                           correctly detected.
    ** 078 GES 20080519   @38302 Disable the special '.' EOF marker when
    **                           parsing a string from the readFieldWorker.
    ** 079 SIY 20080721   @39151 Updated to accommodate changes in logical
    **                           toString() behavior.
    ** 080 GES 20080730   @39236 Fix for setPosition() to handle a new
    **                           position located past the end of the file
    **                           in the same way as the 4GL. The previous
    **                           version incorrectly handled this.
    ** 081 NVS 20080812   @39373 Added support for redirected editors output
    **                           implemented in new writeControlledSpecial()
    **                           method. Refactored shared code into worker
    **                           methods.
    ** 082 SVL 20090208   @41268 readField(FieldReference) method added.
    ** 083 SVL 20090209   @41274 generateFormattedText() behavior fixed.
    ** 084 SIY 20090220   @41340 Added isTerm() method which by default return
    **                           false (means 'not a terminal').
    ** 085 GES 20090422   @41915 Converted to standard string formatting.
    ** 086 GES 20090424   @41973 Import change.
    ** 087 GES 20090501   @42090 Notify listeners about stream end-of-life.
    ** 088 SVL 20090225   @41368 Into generateFormattedText() changed hardcoded
    **                           length of default format for the NumberType to a
    **                           dynamic one, in order to support recid type.
    ** 089 SVL 20090612   @42673 Added "unbuffered" flag. Stop condition (rather
    **                           than error condition) is raised on output errors.
    **                           Notify BufferSizeManager about PUT operations.
    ** 090 GES 20090624   @42885 Protect stream reads, writes and other state
    **                           changes/inspections from closed streams.
    ** 091 ECF 20090709   @43127 Fixed memory leak. Made pageBreakListeners and
    **                           closeListeners sets into WeakHashMaps to ensure
    **                           unreferenced listeners are garbage collected in
    **                           the absence of an explicit deregistration
    **                           mechanism. Note that this changes the iteration
    **                           order of these collections from FIFO (since these
    **                           were previously implemented as LinkedHashSets) to
    **                           undefined. However, this appears to be safe.
    ** 092 SVL 20090815   @43622 Skip multiple sequential delimiters in
    **                           readFieldWorker() only if delimiter is space.
    **                           Parse empty strings as integer, decimal or date
    **                           without any errors.
    ** 093 SIY 20090907   @43834 Explicitly closed stream should not complain about
    **                           writes into closed stream.
    ** 094 SIY 20090925   @44034 Added pendingRowCount memebr and accessor. Now
    **                           getNextLineNum() takes pending rows into account.
    ** 095 SVL 20110403          Added write(byte[]) function.
    ** 096 GES 20110812          Added readChunk() method.
    ** 097 ECF 20121219          Added skipLoneHyphenInput method and modified
    **                           readFieldWorker to make skipping a lone hyphen
    **                           encountered on input dependent upon the return
    **                           value from this new method. This is meant to
    **                           allow subclasses to override this default
    **                           behavior.
    ** 098 EVL 20130116          Added indexOfNewLine() method to make finding
    **                           the new line separator strings the system neutral.
    **                           The method readLineCleanup has been changed to
    **                           use this tool.
    ** 099 CS  20130319          Added conversion support for Landscape and Portrait IO options.
    ** 100 OM  20130406          Added support for 64 bit operations.
    ** 101 GES 20130523          Matched signature changes in BinaryData and used a more efficient
    **                           approach than a byte-by-byte writing loop.
    ** 102 OM  20130607          Fixed datetime/-tz deserialization (fromLiteral).
    ** 103 CA  20131013          Added no-op deleted() method, required by the changes in Finalizable
    **                           interface.
    ** 104 EVK 20140207          In generateFormattedText() added special case for RAW type.
    ** 105 EVL 20140418          Fixes for different format variations.
    ** 106 EVK 20140523          Added unbuffered stream operation.
    ** 107 CA  20140730          Protected stream close/cleanup code from raised conditions, to allow
    **                           close/cleanup fully complete even if conditions were raised.
    ** 108 GES 20141030          Added setConvertSource/Target() methods.
    ** 109 ECF 20150715          Replace StringBuffer with StringBuilder.
    ** 110 EVL 20160224          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
    ** 111 EVK 20161123          Disable formatting for put unformatted output.
    ** 112 OM  20161209          Fixed needsPageBreak() computation. Upgrade code to Java8.
    ** 113 OM  20170612          Granted protected access to [binary] field. Fixed readChar() in
    **                           non-binary case.
    ** 114 CA  20170827          Added stub for readField(ImportFieldExpansion).
    **     HC  20170904          Modified conversion of IMPORT statement to expand extent fields
    **                           dynamically. This should prevent the Java source file size to get out
    **                           of control for large extents.
    ** 115 HC  20171024          Runtime support for SYSTEM-DIALOG PRINT-SETUP and OUTPUT TO PRINTER.
    ** 116 ECF 20171026          Added write(byte[], int, int) method. Refactored readField to make
    **                           core data assignment accessible statically.
    ** 117 EVL 20180620          Adding pulse on close flag support.
    ** 118 EVL 20180831          Fix for NPE when concurrent thread attempts to close the buffer after
    **                           short message when main worker is still in progress.
    ** 119 ECF 20180818          Avoid use of expensive String.format.
    ** 120 SBI 20181106          Changed generateFormattedText in order to format a number according
    **                           to its format.
    ** 121 EVL 20190423          Adding runtime support for stream input-output code page conversions.
    **                           Now by default we do conversion from -cpinternal to -cpstream for
    **                           output and from -cpstream to -cpinternal for input operations.
    **                           Adding MAP/NO-MAP options flag runtime supprort.
    **     GES 20190509          Reworked code to cache cp conversion decision (for performance) and
    **                           to remove code duplication.
    ** 122 CA  20190511          Added support for IMPORT ... UNFORMATTED {record|field|OO property}.
    ** 123 CA  20190906          Improved performance of indexOfNewLine - it searches only the last
    **                           two characters, and not the entire string buffer.
    ** 124 CA  20200211          Allow fractional seconds (convert to millis based on the legacy
    **                           version).
    ** 125 RFB 20200419          Added support for KEEP-MESSAGES to be able to suppress some messages
    **                           from the default display.
    **     CA  20200427          Added support for blob and clob fields.
    ** 126 RFB 20200710          indexOfNewLine was not taking into account when the only character in the buffer
    **                           is the newline itself. It was returning -1, which indicates not found.
    **     GES 20200820          Added some comments.
    **     CA  20210304          Fixed date, datetime and datetimetz literal parsing.
    **     EVL 20210309          Adding raw data type as valid destination for readLine(BDT) method for case when
    **                           we read text file in binary mode into raw buffer variable.
    **     EVL 20210310          Adding more importers for longchar, blob, clob, memptr, recid and rowid data
    **                           types to be able to read into via readLine(BDT) method.  Fixed String line import
    **                           into binary based data types.
    **     EVL 20210311          Small changes to improve compatibility for binary based data types import with
    **                           readLine(BDT) method calling to read binary data file.
    **     CA  20210317          Improved performance of the readBlock(memptr) API.
    **     OM  20210328          OE quirk: if last line is not EOL terminated, the readLn() method returns the
    **                           previous read line (actually, if that EOL is missing the file is binary by
    **                           definition).
    **     OM  20210401          Reverting previous commit. It seems it is not the correct way to handle it.
    **     EVL 20210402          Fix for raw variable set up bug while readLine().  We need to make raw final
    **                           variable size matching the line length to avoid \0 chars keeeping in final raw.
    **                           Also experimental feature to add new line chars when reading raw via String.
    **     EVL 20210405          Improved readLine() processing for raw data type.  All line are now reading.
    **                           The new line separators are embedding into final data buffer.  Taking into
    **                           account different values of initial raw variable size and incoming data length.
    **                           Also OS behind and binary mode is considering to make the final data buffer.
    **     EVL 20210406          Fixed the bug when partial reading into raw variable drops any further reading
    **                           from the current position of the same stream into another variable.
    **     CA  20210422          Reworked 'IMPORT UNFORMATTED raw/memptr/blob' - although it relies on the
    **                           readLine() API in the conversion, the runtime will read the stream byte-by-byte,
    **                           and not 'per-line'.
    **     CA  20210531          Fixed a NPE in 'IMPORT UNFORMATTED lob' - the bytes need to be written, not the
    **                           'result'.
    **     OM  20210602          Support for multibyte character sets. Use CPSTREAM for default char encoding.
    **     CA  20210709          Added APIs to read pristine bytes.  Characters are returned and decoded only by
    **                           the readCh() and readLine() APIs.
    **     CA  20210919          When reading from a file, the "?" literal must be treated as a string, and the ?
    **                           literal must be treated as an unknown value.
    **     CA  20210920          Fixed a regression in reading a line (largest separator needs to be checked first).
    **                           Hard-coded the possible new lines instead of computing them.
    **     CA  20211001          Do not flush file resources on PUT/EXPORT statements.
    **     AL2 20211217          Byte 26 should stop reading only in non-binary Windows.
    **     EVL 20220325          Base refactoring for Stream based classes getting single byte and array of bytes.
    **     IAS 20220408          Fixed support for PUT CONTROL <raw>.
    **     CA  20220515          memptr and library calls are supported on server-side, too.  Added TODO's for
    **                           client-side APIs which will need to call back to server-side, when memptr is used
    **                           as argument.
    **     HC  20220824          Stream now initiates flush when there is buffered data in the related
    **                           RedirectedTerminal. This prevents data directly written to the Stream with
    **                           PUT statement to be written to the target sooner than the buffered data in
    **                           RedirectedTerminal.
    **     EVL 20221207          Adding method to set up LOB directory for Stream object.
    **     EVL 20221212          The runtime implementation is only in FileStream class.
    **     EVL 20221222          Adding runtime support for LOB fields export.  Merges changes from OM for
    **                           readField() method.
    ** 128 PMP 20240403          Added import unformatted flag to readNextLine method.
    **     GES 20240412          Minor clarification in javaodc.
    ** 129 PMP 20240509          Modified putWorker method to check for null field entries.
    ** 130 SB  20240617          Use isWindows instead of recalling EnvinronmentOps.isUnderWindows.
    ** 131 GBB 20240912          Replace ThinClient access to streams with StreamDaemon from OSResourceManager.
    ** 132 FER 20241118          Added the option to print to console if PUT is performed in batched mode. putWorker()
    **                           method. This is done through the new method writeInline().
    **                           Added import OutputManager.
    ** 133 FER 20250304          Changed the if from putWorker to make sure that Window batch processes only print to
    **                           stdout when not redirected to files.
    ** 134 AI  20250228          Improved performance of readLine by buffering the read of bytes.
    ** 135 ES  20250424          Handle StopConditionException through ErrorManager.
    ** 136 AI  20250528          Fixup extra read of byte when max length of bytes was read.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util;

    import java.io.*;
    import java.nio.charset.*;
    import java.util.*;
    import com.goldencode.p2j.ui.LogicalTerminal;
    import com.goldencode.p2j.ui.client.OutputManager;
    import com.goldencode.p2j.ui.client.chui.driver.batch.BatchHelper;
    import com.goldencode.p2j.util.osresource.*;
    import com.goldencode.util.*;
    import com.goldencode.p2j.ui.chui.RedirectedTerminal;
    import com.goldencode.p2j.ui.chui.ThinClient;
    import com.goldencode.p2j.persist.FieldReference;
    import com.goldencode.p2j.schema.*;

    /**
     * A stream class supporting methods for the Progress-compatible reading and
     * writing as well as facilities for line and page processing.  This class
     * defines a common API for read/write operations while implementing the
     * Progress semantics that are common to all streams.  In particular, the
     * formatting, options processing and all knowledge of the wrapper data
     * types is limited to this class.
     * <p>
     * Subclasses implement the specific read/write/seek workers that are
     * specific to the type of stream they represent.  These workers generally
     * pass exceptions through to the caller and this class handles those
     * exceptions that can be handled in a standard manner.  The design is to
     * ensure that the worker methods can be standardized and can be implemented
     * with little effort.  The following methods must be implemented:
     * <p>
     * <pre>
     * {@link #assign}
     * {@link #available}
     * {@link #getPos}
     * {@link #setPos}
     * {@link #getLen}
     * {@link #setLen}
     * {@link #writeByte}
     * {@link #writeCh}
     * {@link #write}
     * {@link #readCh}
     * {@link #readLn}
     * {@link #closeIn}
     * {@link #closeOut}
     * {@link #close}
     * </pre>
     * <p>
     * Please note that many of these abstract methods may simply throw the
     * <code>UnsupportedOperationException</code> if the underlying stream does
     * not support the operation in question.
     * <p>
     * Language statements that are supported:
     * <p>
     * <pre>
     * Language Statement      Java Equivalent
     * ---------------------   ----------------------
     * define stream           {@link #Stream} default constructor
     * export                  {@link #export} (uses {@link #writeField} or {@link #writeBlock})
     * import                  {@link #readField} or {@link #readLine} or {@link #readBlock}
     * input close             {@link #closeIn}
     * input from              Stream subclass constructor
     * input-output close      {@link #close}
     * output close            {@link #closeOut}
     * output to               Stream subclass constructor
     * page                    {@link #advancePage}
     * put                     {@link #put} or {@link #putControl} or {@link #putUnformatted} (uses {@link #putField})
     * readkey                 {@link #readChar}
     * seek                    {@link #setPosition}
     * </pre>
     * <p>
     * Built-in functions that are supported:
     * <p>
     * <pre>
     * Function                Java Equivalent
     * ---------------------   ----------------------
     * line-counter            {@link #getNextLineNum}
     * page-number             {@link #getPageNum}
     * page-size               {@link #getPageSize}
     * seek                    {@link #getPosition}
     * </pre>
     * Language statements that have backing/helper support but which are fully
     * implemented elsewhere:
     * <p>
     * <pre>
     * Language Statement      Uses
     * ---------------------   ----------------------
     * prompt-for              {@link #readFieldWorker}
     * set                     {@link #readFieldWorker}
     * update                  {@link #readFieldWorker}
     * </pre>
     * <p>
     * Direct use of the {@link #putField}, {@link #writeField} and
     * {@link #writeBlock} output methods generate output to a buffer which must
     * first be initialized by {@link #startOutput}.  Once initialized, any number
     * of calls to such output methods may be done and all output will be gathered
     * into the same output buffer.  To write this buffer to the stream, call
     * {@link #endOutput}. Calls to the output methods must never be done outside
     * of a bracketed pair of matching <code>startOutput</code> and
     * <code>endOutput</code> calls otherwise the results will be undefined.
     * <p>
     * Within a given bracketed pair of matching <code>startOutput</code> and
     * <code>endOutput</code> calls, please do not mix the output of
     * <code>putField</code> and <code>writeField</code>.  The results are
     * undefined.
     * <p>
     * Use of {@link #put}, {@link #putControl}, {@link #putUnformatted} and
     * {@link #export} DO NOT require the use of {@link #startOutput} or
     * {@link #endOutput} since they all handle buffering internally.
     * <p>
     * This class honors silent error mode (see {@link ErrorManager}) if this
     * has been enabled.  Otherwise, the end or error conditions are raised
     * (the appropriate exceptions are thrown). <b>At this time, the error text
     * and error numbers are not fully implemented (many of the Progress
     * compatible values are unknown). In addition, the state of the stream
     * may be different from the Progress behavior in cases where an error is
     * raised.</b>
     *
     * @author    GES
     */
    public abstract class Stream
    implements Finalizable
    {
       /**
        * The value returned by {@link #readFieldWorker(BaseDataType)} when the exact <code>?</code> character
        * was read.
        */
       public static final Object UNKNOWN_VALUE = new Object();

       /** Enum for representing the stream print modes*/
       public static enum PrintMode { LANDSCAPE, PORTRAIT, NONE };

       /** The default delimiter to use for field level record reading. */
       public static final char DELIM = ' ';

       /** The line separator character(s) for the current platform. */
       public static final String NEWLINE = System.getProperty("line.separator");

       /** The length if line separator. */
       public static final int NEWLINE_LEN = NEWLINE.length();

       /**
        * The new lines, ordered by their length.  The order is important as a string with '\r\n' must match
        * with '\r\n' and not '\n'.
        */
       public static final String[] NEW_LINES = new String[] { "\r\n", "\n" };

       /** Special value which triggers setting of default number of lines per page in paged mode. */
       public static final int SET_PAGED_DEFAULT = -1;

       /** The default number of lines per page when in paged mode (56). */
       public static final int PAGED_DEFAULT = 56;

       /** The <code>EOF</code> character. */
       public static final char EOF = (char) 0x1A;

       /** The Carriage Return character {@code CR = 0x0D}. */
       public static final int CR = 0x0D;

       /** The Line Feed character {@code LF = 0x0A}. */
       public static final int LF = 0x0A;

       /** The Carriage Return character and Line Feed character */
       public static final String CRLF = "\r\n";

       /** {@link #putWorker} mode for a standard PUT statement. */
       public static final int MODE_PUT_NORMAL = 0;

       /** {@link #putWorker} mode for a PUT CONTROL statement. */
       public static final int MODE_PUT_CONTROL = 1;

       /** {@link #putWorker} mode for a PUT UNFORMATTED statement. */
       public static final int MODE_PUT_UNFORMATTED = 2;

       /** {@link #putWorker} mode for an EXPORT statement. */
       public static final int MODE_EXPORT = 3;

       /** Message that indicates that pipe to subprocess has been broken. */
       public static final String PIPE_BROKEN_MESSAGE = "** Pipe to subprocess has been broken";

       /** Number of milliseconds to pause between checks for input. */
       protected static final int increment = 200;

       /** Flag denoting if input resources have already been cleaned up. */
       protected boolean doneIn = false;

       /** Flag denoting if output resources have already been cleaned up. */
       protected boolean doneOut = false;

       /** External option value for standard character translations occur or not. */
       protected boolean convert = true;

       /** Internal value. Determines if standard character translations occur or not. */
       protected boolean _convert = false;

       /** Determines if codepage conversion decision is cached. */
       protected boolean convertCached = false;

       /** The Legacy source codepage for character conversion. */
       protected String sourceCp = null;

       /** The current source charset. This is never {@code null}. */
       protected Charset sourceCharset;

       /** Target codepage for character conversion.  Is {@code null} if no conversion was specified. */
       protected String targetCp = null;

       /** The current source charset. Is {@code null} if no conversion was specified. */
       protected Charset targetCharset;

       /** Default value of the -cpstream option. */
       protected static final String streamCp = I18nOps._getCPStream();

       /** Default value of the -cpinternal option. */
       protected static final String internalCp = I18nOps._getCPInternal();

       /** Determines if control character translations occur or not. */
       protected boolean binary = false;

       /** Flag to keep NO-MAP stream option. */
       protected boolean noMap = false;

       /** Variable to keep MAP stream option. */
       protected String mapEntryProtermcap = null;

       /** Current page number for the output stream. */
       private int pageNum = 0;

       /** Lines of output per page or 0 for non-paged output.  */
       private int pageSize = 0;

       /** Line number of the next line to be output on the current page. */
       private int nextLineNum = 1;

       /** Current input line being processed when reading fields. */
       private String currentIn = null;

       /** Current index position into the line being read. */
       private int indexIn = 0;

       /** Current output line being processed when writing fields. */
       private StringBuilder currentOut = null;

       /** Flag indicating active put worker operation to avoid buffer closing. */
       private boolean putWorkerActive = false;

       /** Lock to sync the putWorkerActive access. */
       private final Object outActivityLock = new Object();

       /** Current column position on the line being written. */
       private int column = 0;

       /** Flag denoting if the output buffer is currently being used. */
       private boolean active = false;

       /** The print display mode of the stream */
       private PrintMode displayMode = PrintMode.NONE;

       /** Flag denoting if the output is unformatted. */
       private boolean unformatted = false;

       /** Flag denoting if the UI functions that read input should echo it. */
       private boolean echo = true;

       /** Flag denoting if output modifies line/page counters. */
       private boolean control = false;

       /** Flag to detect if a dropped data occurred in line reading. */
       private boolean nullQuirk = false;

       /** Character used for delimiting import and export fields. */
       private char delim = DELIM;

       /** List of headers that are to be output (valid in paged mode). */
       private Set<RepeatedOutput> headers = null;

       /** Size of current headers to be output in lines. */
       private int headerSize = 0;

       /** List of footers are to be output (valid in paged mode). */
       private Set<RepeatedOutput> footers = null;

       /** Size of current footers to be output in lines. */
       private int footerSize = 0;

       /** Intermediate buffer of redirected terminal output. */
       private ArrayList<String> buffer = null;

       /** Nesting level for {@link #flushControlled} method invocations. */
       private int flushLevel = 0;

       /** Objects to receive a notification when a page break is written. */
       private Map<PageBreakListener, Object> pageListeners = null;

       /** Objects to receive a notification when a closeOut is issued. */
       private Map<CloseListener, Object> closeListeners = null;

       /** Objects to receive a notification when a page advance is issued. */
       private Set<AdvanceListener> advanceListeners = null;

       /** Objects to receive a notification when a flush is issued. */
       private Set<FlushListener> flushListeners = null;

       /** Flag indicating headers being output */
       private boolean inHeaders = false;

       /** Track trailing NEWLINE state */
       private boolean trailingNL = true;

       /** Track trailing NEWLINE state */
       private boolean trailingWithoutPutNL = true;

       /** Field indicating what page had output last */
       private int lastPageWithOutput = -1;

       /**
        * The flag saves the put parameter in the putWorker method; used if we
        * need to restart the output.
        */
       private boolean savedPut = false;

       /** Flag indicating a PUT statement is being used. */
       private boolean isPut = false;

       /**
        * When the flag is true, the page-footers were emitted.
        */
       private boolean footersEmitted = false;

       /**
        * While this flag is set, the current page will be written to ignoring
        * a page overflow condition.
        */
       private boolean contiguous = false;

       /** If <code>true</code>, this is an unnamed stream. */
       private boolean unnamed = false;

       /**
        * When the flag is true due to editor(s), the next form feed will be skipped.
        */
       private boolean suppressFormFeed = false;

       /** Determines whether the stream is UNBUFFERED */
       private boolean unbuffered = false;

       /** Name of the stream variable used in the legacy 4GL source code. */
       private String name = null;

       /** Number of rows pending in RemoteTerminal. */
       private int pendingRowCount = 0;

       /** Flag to mark pulse on close action. */
       private boolean pulseOnClose = false;

       /** Flag denoting if error, warning and message statements are suppressed. */
       private boolean keepMessages = false;

       /** Check if we are under Windows OS. */
       protected final boolean isWindows = EnvironmentOps.isUnderWindowsFamily();

       /** readByte batch size */
       private final int BATCH_SIZE = 1024;

       /**
        * Flag to direct this stream to call {@linkplain #flush()} in {@link #putWorker(FieldEntry[], int, char)}.
        */
       private boolean flushBeforePut;

       /**
        * Default constructor which does nothing.
        */
       public Stream()
       {
          this(null);
       }

       /**
        * Construct an instance associated with a specific legacy 4GL variable name.
        *
        * @param    name
        *           The legacy variable name.
        */
       public Stream(String name)
       {
          this.name = name;
          setConvertSource(streamCp); // always starting with CPSTREAM as default for stream encoding/decoding
       }

       /**
        * Assign a value derived from a string into a data wrapper variable.
        *
        * @param   var
        *          Variable into which datum is assigned.
        * @param   value
        *          String value of the datum to be assigned.
        * @param   iso
        *          The {@code value} string is in ISO mode.
        */
       public static void assignDatum(BaseDataType var, String value, boolean iso)
       {
          if (var != null && value != null)
          {
             boolean emptyRes = value.isEmpty();

             if (var instanceof blob)
             {
                if (!emptyRes)
                {
                   ((blob) var).assign(value.getBytes());
                }
             }
             else if (var instanceof clob)
             {
                ((clob) var).assign(emptyRes ? "" : new character(value));
             }
             else if (var instanceof integer)
             {
                // integer must come before int64. Otherwise the operator instanceof will match
                // the integer instances, too
                ((integer) var).assign(emptyRes ? new integer(0) : new integer(value));
             }
             else if (var instanceof int64)
             {
                ((int64) var).assign(emptyRes ? new int64(0) : new int64(value));
             }
             else if (var instanceof decimal)
             {
                ((decimal) var).assign(emptyRes ? new decimal(0) : new decimal(value));
             }
             else if (var instanceof datetimetz)
             {
                // datetimetz must come before datetime and date Otherwise the operator instanceof
                // will match the datetimetz instances, too
                if (emptyRes)
                {
                   var.setUnknown();
                }
                else
                {
                   ((datetimetz) var).assign(
                         iso ? datetimetz.parseLiteral(value) : datetimetz.fromLiteral(value));
                }
             }
             else if (var instanceof datetime)
             {
                // datetime must come before date Otherwise the operator instanceof will match the
                // datetime instances, too
                if (emptyRes)
                {
                   var.setUnknown();
                }
                else
                {
                   ((datetime) var).assign(
                         iso ? datetime.parseLiteral(value) : datetime.fromLiteral(value));
                }
             }
             else if (var instanceof date)
             {
                if (emptyRes)
                {
                   var.setUnknown();
                }
                else
                {
                   ((date) var).assign(iso ? date.parseLiteral(value) : new date(value));
                }
             }
             else if (var instanceof logical)
             {
                ((logical) var).assign(new logical(value));
             }
             else if (var instanceof character)
             {
                ((character) var).assign(new character(value));
             }
             else if (var instanceof raw)
             {
                ((raw) var).assign(new raw(value));
             }
          }
       }

       /**
        * Accesses the line number of the next line to be output on the current page.
        *
        * @return   The next line number to be output.
        */
       public integer getNextLineNum()
       {
          if (protectWrites())
          {
             return new integer(0);
          }

          return new integer(rawGetNextLineNum() + getPendingRowCount());
       }

       /**
        * Accesses the line number of the next line to be output on the current page.
        *
        * @return   The next line number to be output.
        */
       public int rawNextLineNum()
       {
          return rawGetNextLineNum();
       }

       /**
        * Accesses the current page number in an output stream.  The return value
        * is undefined for an input stream.
        *
        * @return   The current page number.
        */
       public integer getPageNum()
       {
          if (protectWrites())
             return new integer(0);

          return new integer(rawGetPageNum());
       }

       /**
        * Accesses the number of lines per page in an output stream, where 0
        * indicates that this stream is not paged.  The return value is undefined
        * for an input stream.
        *
        * @return   The number of lines per page of output or 0 if the output
        *           is not paged.
        */
       public integer getPageSize()
       {
          if (protectWrites())
             return new integer(0);

          return new integer(rawGetPageSize());
       }

       /**
        * Accesses the number of lines per page in an output stream, where 0
        * indicates that this stream is not paged.  The return value is undefined
        * for an input stream.
        *
        * @return   The number of lines per page of output or 0 if the output
        *           is not paged.
        */
       public int rawPageSize()
       {
          return rawGetPageSize();
       }

       /**
        * Sets the number of lines per page in an output stream, where 0
        * indicates that this stream is not paged.
        *
        * @param    size
        *           The number of lines per page of output or 0 if the output
        *           is not paged.  If the input value is <code>unknown</code>
        *           no change will be made.
        */
       public void setPageSize(NumberType size)
       {
          if (!size.isUnknown())
          {
             setPageSize(size.intValue());
          }
       }

       /**
        * Sets the number of lines per page in an output stream, where 0
        * indicates that this stream is not paged.
        *
        * @param    size
        *           The number of lines per page of output or 0 if the output
        *           is not paged.
        */
       public void setPageSize(double size)
       {
          int sz = (int) size;

          rawSetPageSize((sz <= 0) ? 0 : sz);
       }

       /**
        * Sets the number of lines per page in an output stream to the
        * {@link #PAGED_DEFAULT} or to terminal height for {@link TerminalStream}.
        */
       public void setPageSize()
       {
          rawSetPageSize(SET_PAGED_DEFAULT);
       }

       /**
        * Sets the stream into binary mode which disables character translations
        * during {@link #readChar}.  It is important to note that field and line
        * reading (<code>import</code>, <code>set</code>, <code>update</code>
        * <code>prompt-for</code> and <code>import unformatted</code>) are
        * unaffected by this setting (this is based on Progress testcases).
        */
       public void setBinary()
       {
          binary = true;
       }

       /**
        * Sets the stream NO-MAP option causing to completely ignore PROTERMCAP file while data
        * transformation.
        */
       public void setNoMap()
       {
          noMap = true;
       }

       /**
        * Sets the stream MAP option causing to use PROTERMCAP file entry while data transformation.
        */
       public void setMap(String protermcapEntry)
       {
          // TODO: implement the real work
          mapEntryProtermcap = protermcapEntry;
       }

       /**
        * Sets the stream's character conversion mode.
        *
        * @param    convert
        *           The new character conversion mode.  <code>true</code> to
        *           enable character conversion, <code>false</code> to disable
        *           conversion.
        */
       public void setConvert(boolean convert)
       {
          this.convert = convert;
       }

       /**
        * Gets the stream's character conversion mode.
        *
        * @return   The convert mode. <code>true</code> means that character
        *           conversion is enabled.
        */
       public boolean getConvert()
       {
          return convert;
       }

       /**
        * Sets the stream's source codepage and activates character conversion mode.
        *
        * @param   cp
        *          The source codepage for character conversion mode.
        */
       public void setConvertSource(String cp)
       {
          // skip for null
          if (cp != null)
          {
             // protect from setting to empty or just whitespaced strings
             cp = cp.trim();
             if (!cp.isEmpty())
             {
                sourceCp = cp;
             }
          }
          else
          {
             sourceCp = I18nOps.ABL_LATIN1; // default to Latin-1 / ISO-8859-1
          }

          String javaCS = I18nOps.getJavaCharset(sourceCp);
          if (javaCS == null)
          {
             javaCS = I18nOps.JAVA_LATIN1; // TODO: maybe throw exception instead
             sourceCp = I18nOps.ABL_LATIN1; // to reflect the actual value
          }

          this.sourceCharset = Charset.forName(javaCS);
       }

       /**
        * Sets the stream's source codepage and activates character conversion mode.
        *
        * @param    cp
        *           The source codepage for character conversion mode.
        */
       public void setConvertSource(character cp)
       {
          if (cp == null || cp.isUnknown())
          {
             UnimplementedFeature.todo("CONVERT SOURCE: confirm unknown handling is correct.");
             return;
          }

          setConvertSource(cp.toStringMessage());
       }

       /**
        * Sets the stream's target codepage and activates character conversion mode.
        *
        * @param    cp
        *           The target codepage for character conversion mode.
        */
       public void setConvertTarget(String cp)
       {
          // skip for null
          if (cp != null && !cp.trim().isEmpty())
          {
             // protect from setting to empty or just whitespaced strings
             cp = cp.trim();
             if (!cp.isEmpty())
             {
                targetCp = cp;
             }
          }
          else
          {
             targetCp = I18nOps.ABL_LATIN1; // default to Latin-1 / ISO-8859-1
          }

          String javaCS = I18nOps.getJavaCharset(targetCp);
          if (javaCS == null)
          {
             javaCS = I18nOps.JAVA_LATIN1; // TODO: maybe throw exception instead
             targetCp = I18nOps.ABL_LATIN1; // to reflect the actual value
          }

          this.targetCharset = Charset.forName(javaCS);
       }

       /**
        * Sets the stream's target codepage and activates character conversion mode.
        *
        * @param    cp
        *           The target codepage for character conversion mode.
        */
       public void setConvertTarget(character cp)
       {
          if (cp == null || cp.isUnknown())
          {
             UnimplementedFeature.todo("CONVERT TARGET: confirm unknown handling is correct.");
             return;
          }

          setConvertTarget(cp.toStringMessage());
       }

       /**
        * Sets the stream's lob directory located as a subdirectory for opened current file stream directory.
        *
        * @param    lobDir
        *           The name for subdirectory to store LOB files.
        */
       public void setLobDirectory(String lobDir)
       {
          // nothing to do by default only FileStream has the real implementation
       }

       /**
        * Sets the stream's lob directory located as a subdirectory for opened current file stream directory.
        *
        * @param    lobDir
        *           The name for subdirectory to store LOB files.
        */
       public void setLobDirectory(character lobDir)
       {
          if (lobDir == null || lobDir.isUnknown())
          {
             UnimplementedFeature.todo("LOB-DIR: confirm unknown handling is correct.");
             return;
          }

          setLobDirectory(lobDir.toStringMessage());
       }

       /**
        * Sets the stream into echo mode which causes UI statements that read
        * data from a stream to echo that data to the current output destination.
        * If this is set on an output stream, then if the input stream also has
        * echo set to <code>true</code> then the output will be echoed, but if
        * either stream has echo as <code>false</code>, no echoing will be done.
        * Note that this only affects UI statements that are not operating
        * against the terminal.
        *
        * @param    echo
        *           The new echo mode.
        */
       public void setEcho(boolean echo)
       {
          this.echo = echo;
       }

       /**
        * Gets the stream's echo mode which causes UI statements that read
        * data from a stream to echo that data to the current output destination.
        * If this is set on an output stream, then if the input stream also has
        * echo set to <code>true</code> then the output will be echoed, but if
        * either stream has echo as <code>false</code>, no echoing will be done.
        * Note that this only affects UI statements that are not operating
        * against the terminal.
        *
        * @return   The echo mode.
        */
       public boolean getEcho()
       {
          return echo;
       }

       /**
        * Gets the stream's unformatted mode flag.
        *
        * @return   If <code>true</code>, all output is in a raw unformatted
        *           mode unless overridden by an explicit format string.
        */
       public boolean getUnformatted()
       {
          return unformatted;
       }

       /**
        * Sets the stream into unformatted mode which disables default format
        * processing in {@link #putField}.
        *
        * @param    fmt
        *           If <code>true</code>, all output is in a raw unformatted
        *           mode unless overridden by an explicit format string.
        */
       public void setUnformatted(boolean fmt)
       {
          unformatted = fmt;
       }

       /**
        * Gets the stream's control mode flag which defines whether subsequent
        * characters written to the stream will affect line/page counters.
        * <p>
        * Warning: in Progress this is described as ensuring that the "position"
        * (presumably the write position) will not be changed when this option
        * is enabled.  However, this really does not make any sense in the case
        * where output characters are used for control purposes.  No support is
        * provided for this case.
        *
        * @return   If <code>true</code>, all output is emitted without
        *           modification of line/page counters.
        */
       public boolean getControl()
       {
          return control;
       }

       /**
        * Sets the stream into control mode which ensures that subsequent
        * characters written to the stream will not affect line/page counters.
        * <p>
        * Warning: in Progress this is described as ensuring that the "position"
        * (presumably the write position) will not be changed when this option
        * is enabled.  However, this really does not make any sense in the case
        * where output characters are used for control purposes.  No support is
        * provided for this case.
        *
        * @param    mode
        *           If <code>true</code>, all output is emitted without
        *           modification of line/page counters.
        */
       public void setControl(boolean mode)
       {
          column = 0;
          control = mode;
       }

       /**
        * Gets the stream's field import/export delimiter character.
        *
        * @return   The current delimiter <code>char</code> (defaults to
        *           {@link #DELIM}.
        */
       public char getDelimiter()
       {
          return delim;
       }

       /**
        * Sets the stream's delimiter character for import/export of fields.
        *
        * @param    delim
        *           The current delimiter <code>char</code>.
        */
       public void setDelimiter(char delim)
       {
          this.delim = delim;
       }

       /**
        * Sets the stream's <code>keepMessages</code> mode which causes certain UI statements that write
        * data to the default window to be suppressed when outpu is being redirected.
        * The types of UI statements are 4GL errors and warnings, and UI statements written
        * using MESSAGE.
        */
       public void setKeepMessages()
       {
          keepMessages = true;
       }

       /**
        * Set stream display mode to landscape
        */
       public void setLandscape()
       {
          displayMode = PrintMode.LANDSCAPE;
       }

       /**
        * Set stream display mode to portrait
        */
       public void setPortrait()
       {
          displayMode = PrintMode.PORTRAIT;
       }

       /**
        * Returns the display mode of the stream.
        * <p>
        * Possible values: LANDSCAPE, PORTRAIT and NONE.
        *
        * @return The display mode for this stream as one of the <code>PrintMode</code>
        *         enum values.
        */
       public PrintMode getDisplayMode()
       {
          // TODO use this in  Stream implementations for LANDSCAPE and PORTRAIT logic
          return displayMode;
       }

       /**
        * Checks this stream redirection type: named or unnamed.
        *
        * @return   <code>true</code> if this is an unnamed stream.
        */
       public boolean isUnnamed()
       {
          return unnamed;
       }

       /**
        * Marks this stream as being named or unnamed.
        *
        * @param    unnamed
        *           If <code>true</code>, then this stream is unnamed.
        */
       public void setUnnamed(boolean unnamed)
       {
          this.unnamed = unnamed;
       }

       /**
        * Makes the stream UNBUFFERED.
        */
       public void setUnbuffered()
       {
          unbuffered = true;
       }

       /**
        * Determines whether the stream is UNBUFFERED.
        *
        * @return <code>true</code> if the stream is UNBUFFERED.
        */
       public boolean isUnbuffered()
       {
          return unbuffered;
       }

       /**
        * Sets the stream's delimiter character for import/export of fields.
        *
        * @param    delim
        *           The current delimiter <code>char</code> (if this is not
        *           a single character, only the first character will be used).
        *           If <code>null</code> or an empty string, the delimiter is
        *           reset to the default of {@link #DELIM}.
        */
       public void setDelimiter(String delim)
       {
          if (delim == null)
          {
             this.delim = DELIM;
          }
          else
          {
             this.delim = delim.length() > 0 ? delim.charAt(0) : DELIM;
          }
       }

       /**
        * Gets the stream's keepMessages mode which causes certain UI statements that write
        * data to the default window to be suppressed when outpu is being redirected.
        * The types of UI statements are 4GL errors and warnings, and UI statements written
        * using MESSAGE.
        *
        * @return   The keepMessages mode.
        */
       public boolean isKeepMessages()
       {
          return keepMessages;
       }

       /**
        * Get the name of the stream variable used in the legacy 4GL source code.
        *
        * @return   The legacy variable name.
        */
       public String getName()
       {
          return name;
       }

       /**
        * Set the name of the stream variable used in the legacy 4GL source code.
        *
        * @param    name
        *           The legacy variable name.
        */
       public void setName(String name)
       {
          this.name = name;
       }

       /**
        * Sets the flush-before-put flag. The method is called to inform the stream to flush any buffered data
        * before PUT writes to the stream. This is needed to flush any external data sources connected to this
        * stream.
        *
        * @param   value
        *          The new value to set.
        */
       public void setFlushBeforePut(boolean value)
       {
          flushBeforePut = value;
       }

       /**
        * Raise an ERROR condition if the caller cannot input from the stream due
        * to the input source being closed or never having been opened.
        *
        * @return   <code>true</code> if the calling code should silently return
        *           which will occur if the condition would have been raised but
        *           wasn't due to NO-ERROR protection (silent error mode). Will
        *           return <code>false</code> if it is OK to read from the stream.
        */
       public boolean protectReads()
       {
          if (!isIn())
          {
             String msg = String.format("Attempt to read from closed stream %s",
                                        (name == null) ? "" : name);
             ErrorManager.recordOrThrowError(1386, msg);
             return true;
          }

          return false;
       }

       /**
        * Raise an ERROR condition if the caller cannot output to the stream due
        * to the output destination being closed or never having been opened.
        *
        * @return   <code>true</code> if the calling code should silently return
        *           which will occur if the condition would have been raised but
        *           wasn't due to NO-ERROR protection (silent error mode). Will
        *           return <code>false</code> if it is OK to write to the stream.
        */
       public boolean protectWrites()
       {
          if (!isOut())
          {
             if (!isSilent())
             {
                String msg = String.format("Attempt to write to closed stream %s",
                                           (name == null) ? "" : name);
                ErrorManager.recordOrThrowError(1387, msg);
             }
             return true;
          }

          return false;
       }

       /**
        * Check if stream is in silent mode.
        *
        * @return  <code>false</code>, by default all streams are verbose.
        */
       public boolean isSilent()
       {
          return false;
       }

       /**
        * Read a character from the stream, blocking until input is received.
        * <p>
        * The exact character read from the input stream will be returned,
        * except that when <code>binary</code> mode is false, all 0x0A characters
        * (line feeds) are converted into a 0x0D (carriage return or a Progress
        * "RETURN" key).
        * <p>
        * The underlying stream subclass determines the content of the result.
        * Byte oriented streams such as pipes or files will return a byte while
        * streams that generate keystrokes or characters may return a DBCS or
        * Unicode character.  Please see the {@link #readCh} method of the
        * subclass to obtain more information.
        *
        * @return   The character read, -1 if there is any interruption or
        *           error during processing or -2 at <code>EOF</code>.
        */
       public integer readChar()
       {
          if (protectReads())
          {
             return new integer(-1);
          }

          int ch = readCh();

          if (!binary)
          {
             if (isWindows)
             {
                if (ch == CR)
                {
                   int lf = peekCh();
                   if (lf == LF)
                   {
                      readCh(); // truly read / 'burn' the LF
                   }
                }
                else if (ch == LF)
                {
                   ch = CR;
                }
             }
             else // on Linux emulation
             {
                if (ch == LF)
                {
                   ch = CR;
                }
             }
          }

          return new integer(ch);
       }

       /**
        * Read a character from the stream, pausing until input is received or
        * the requested number of seconds have elapsed.
        * <p>
        * The exact character read from the input stream will be returned,
        * except that when <code>binary</code> mode is false, all 0x0A characters
        * (line feeds) are converted into a 0x0D (carriage return or a Progress
        * "RETURN" key).
        * <p>
        * The underlying stream subclass determines the content of the result.
        * Byte oriented streams such as pipes or files will return a byte while
        * streams that generate keystrokes or characters may return a DBCS or
        * Unicode character.  Please see the {@link #readCh} method of the
        * subclass to obtain more information.
        *
        * @param    pause
        *           The milliseconds to wait.  If this is negative, the pause is
        *           ignored and the wait is indefinite.  If 0, there is an
        *           immediate return of -1 if no character is available for
        *           reading.
        *
        * @return   The character read, -1 if the timeout occurs or if there is
        *           any interruption or error during processing or -2 at
        *           <code>EOF</code>.
        *
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       @SuppressWarnings("static-access")
       public integer readChar(int pause)
       throws StopConditionException
       {
          if (protectReads())
             return new integer(-1);

          try
          {
             if (pause < 0)
             {
                // indefinite wait for a key press
                return readChar();
             }
             if (pause == 0)
             {
                if (available() > 0)
                {
                   return readChar();
                }
             }
             else
             {
                // timed wait in short increments
                long millis = System.currentTimeMillis();
                long target = millis + pause;

                while (millis <= target)
                {
                   if (available() > 0)
                   {
                      return readChar();
                   }

                   // pause for a short period
                   Thread.currentThread().sleep(increment);

                   // detect how close we are to the end
                   millis = System.currentTimeMillis();
                }
             }
          }
          catch (InterruptedIOException | InterruptedException iioe)
          {
             ErrorManager.handleStopException(new StopConditionException(iioe));
          }
          catch (IOException ioe)
          {
             // silently return error below
          }

          return new integer(-1);
       }

       /**
        * Resets the internal buffer which holds the current line being parsed
        * and the current index into that buffer.  This must be called after
        * all {@link #readField} processing has occurred for a given line.
        * <p>
        * This also resets the current delimiter to the default.
        */
       public void resetCurrentLine()
       {
          setDelimiter(null);
          currentIn = null;
          indexIn   = 0;
       }

       /**
        * Read the fields from the current line buffer using the previously
        * specified delimiter using {@link #setDelimiter} or the default
        * delimiter of {@link #DELIM} before assigning the data into the given
        * variable <code>var</code> elements in the order the data was read. The
        * number of fields read is equal to the number of elements in the array.
        * <p>
        * Parsing is done using {@link #readFieldWorker}.
        * <p>
        * If the current line buffer is empty, a new line will be read from the
        * stream.
        * <p>
        * In the following conditions, no assignment will be made:
        * <ul>
        *   <li> a field is requested after the entire line has already been
        *        processed
        *   <li> an unquoted hyphen is used as the contents of the field AND the
        *        delimiter is NOT a hyphen (the index is still incremented past
        *        the field)
        *   <li> the first char of a field starts with a double quote but there
        *        is no closing quote (the index is incremented to the end of the
        *        buffer)
        *   <li> the <code>var</code> element is <code>null</code> (the caller
        *        wants the field to be read and the index incremented but the
        *        contents are ignored), this is equivalent to {@link #skipField}
        * </ul>
        * <p>
        * After all field reads for a given line are complete,
        * {@link #resetCurrentLine} must be called to enable input for the
        * next line.  Otherwise, all subsequent reads will continue trying to
        * read from the current line. This will result in reading data left on
        * the current line or reading nothing if the line has already been fully
        * read.  <code>resetCurrentLine</code> notifies the stream environment
        * that it should move to processing the next line.
        * <p>
        * This method is used to implement the field level input reading for the <code>IMPORT</code> 4GL
        * language statement.  It does not handle the line reading mode (<code>IMPORT UNFORMATTED</code>).
        *
        * @param    var
        *           The array of elements to which the results are assigned. Must
        *           not be <code>null</code>, but any given element in the array
        *           may be <code>null</code>.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void readField(BaseDataType[] var)
       throws EndConditionException,
              ErrorConditionException
       {
          if (protectReads())
          {
             return;
          }

          for (BaseDataType aVar : var)
          {
             readField(aVar);
          }
       }

       public void readField(ImportFieldExpansion f)
       {
          // TODO:
       }

       /**
        * Read the next field from the current line buffer using the previously
        * specified delimiter using {@link #setDelimiter} or the default
        * delimiter of {@link #DELIM} before assigning the data using the given
        * database field reference <code>ref</code>.
        * <p>
        * Parsing is done using {@link #readFieldWorker}.
        * <p>
        * If the current line buffer is empty, a new line will be read from the
        * stream.
        * <p>
        * In the following conditions, no assignment will be made:
        * <ul>
        *   <li> a field is requested after the entire line has already been
        *        processed
        *   <li> an unquoted hyphen is used as the contents of the field AND the
        *        delimiter is NOT a hyphen (the index is still incremented past
        *        the field)
        *   <li> the first char of a field starts with a double quote but there
        *        is no closing quote (the index is incremented to the end of the
        *        buffer)
        *   <li> the <code>ref</code> reference is <code>null</code> (the caller
        *        wants the field to be read and the index incremented but the
        *        contents are ignored), this is equivalent to {@link #skipField}
        * </ul>
        * <p>
        * After all field reads for a given line are complete,
        * {@link #resetCurrentLine} must be called to enable input for the
        * next line.  Otherwise, all subsequent reads will continue trying to
        * read from the current line. This will result in reading data left on
        * the current line or reading nothing if the line has already been fully
        * read.  <code>resetCurrentLine</code> notifies the stream environment
        * that it should move to processing the next line.
        *
        * @param    ref
        *           The database field reference using which the result is
        *           assigned.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void readField(FieldReference ref)
       throws EndConditionException,
              ErrorConditionException
       {
          if (protectReads())
             return;

          if (ref == null)
          {
             // Skip field.
             readField((BaseDataType) null);
          }
          else
          {
             Integer extent = ref.getExtent();
             if (ref.getIndex() >= 0 || extent == null)
             {
                // a non-extent or indexed extent field reference
                BaseDataType var = ref.get();
                readField(var);

                if (ref.getType() == blob.class)
                {
                   var.assign(PropertyMapper.preprocessBlob(new character(var)));
                }
                else if (ref.getType() == clob.class)
                {
                   var.assign(PropertyMapper.preprocessClob(new character(var)));
                }
                else
                {
                   ref.set(var);
                }
             }
             else
             {
                // an extent field reference, read all individual subscripts
                for (int i = 0; i < extent; i++)
                {
                   readField(new FieldReference(ref, i));
                }
             }
          }
       }

       /**
        * Read the next field from the current line buffer using the previously
        * specified delimiter using {@link #setDelimiter} or the default
        * delimiter of {@link #DELIM} before assigning the data into the given
        * variable <code>var</code>.
        * <p>
        * Parsing is done using {@link #readFieldWorker}.
        * <p>
        * If the current line buffer is empty, a new line will be read from the
        * stream.
        * <p>
        * In the following conditions, no assignment will be made:
        * <ul>
        *   <li> a field is requested after the entire line has already been
        *        processed
        *   <li> an unquoted hyphen is used as the contents of the field AND the
        *        delimiter is NOT a hyphen (the index is still incremented past
        *        the field)
        *   <li> the first char of a field starts with a double quote but there
        *        is no closing quote (the index is incremented to the end of the
        *        buffer)
        *   <li> the <code>var</code> reference is <code>null</code> (the caller
        *        wants the field to be read and the index incremented but the
        *        contents are ignored), this is equivalent to {@link #skipField}
        * </ul>
        * <p>
        * After all field reads for a given line are complete,
        * {@link #resetCurrentLine} must be called to enable input for the
        * next line.  Otherwise, all subsequent reads will continue trying to
        * read from the current line. This will result in reading data left on
        * the current line or reading nothing if the line has already been fully
        * read.  <code>resetCurrentLine</code> notifies the stream environment
        * that it should move to processing the next line.
        *
        * @param    var
        *           The variable to which the result is assigned.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void readField(BaseDataType var)
       throws EndConditionException,
              ErrorConditionException
       {
          if (protectReads())
             return;

          Object result = readFieldWorker(var);
          if (result == UNKNOWN_VALUE)
          {
             var.setUnknown();
             return;
          }

          // the result must be a string or null value
          assignDatum(var, (String) result, false);
       }

       /**
        * Read a block of data from the stream with the block size equal to the
        * size of the binary data array in the given <code>memptr</code> variable.
        * The data will be written into the memptr starting at the first byte
        * in the array.  The read data will be the smaller of the block size or
        * the file size.  Only an error or end of file condition will cause this
        * read to terminate before the block size has been read.
        *
        * @param    var
        *           The variable to which the result is assigned.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void readBlock(memptr var)
       throws EndConditionException,
              ErrorConditionException
       {
          // TODO: memptr may be on server-side, improve this

          if (protectReads())
             return;

          int res = readBlockRemote(var);

          if (res == -1)
          {
             String spec = "Error during read at index %d";

             // TODO: actual Progress compatible text and number are unknown
             ErrorManager.recordOrThrowError(-1, String.format(spec, 0));
             return;
          }
          else if (res == -2)
          {
             throw new EndConditionException("EOF");
          }
       }

       /**
        * Read the next field from the current line buffer using the previously
        * specified delimiter using {@link #setDelimiter} or the default
        * delimiter of {@link #DELIM}.
        * <p>
        * If the current line buffer is empty, a new line will be read from the
        * stream.
        * <p>
        * In the following conditions, a <code>null</code>return value will be
        * provided:
        * <ul>
        *   <li> a field is requested after the entire line has already been
        *        processed
        *   <li> an unquoted hyphen is used as the contents of the field AND the
        *        delimiter is NOT a hyphen (the index is still incremented past
        *        the field)
        *   <li> the first char of a field starts with a double quote but there
        *        is no closing quote (the index is incremented to the end of the
        *        buffer)
        *   <li> the <code>var</code> parameter is <code>null</code> (the caller
        *        wants the field to be read and the index incremented but the
        *        contents are ignored)
        *   <li> an error condition has been raised and silent error mode has
        *        been specified
        * </ul>
        * <p>
        * If the ? literal is found, it will return the {@link #UNKNOWN_VALUE} instance.  Otherwise, for the
        * <code>"?"</code> string, the actual string is returned.
        * <p>
        * After all field reads for a given line are complete,
        * {@link #resetCurrentLine} must be called to enable input for the
        * next line.  Otherwise, all subsequent reads will continue trying to
        * read from the current line. This will result in reading data left on
        * the current line or reading nothing if the line has already been fully
        * read.  <code>resetCurrentLine</code> notifies the stream environment
        * that it should move to processing the next line.
        *
        * @param    var
        *           If <code>null</code> the next field read will be discarded
        *           instead of returned.
        *
        * @return   The string data that was read, the {@link #UNKNOWN_VALUE} or <code>null</code> (see above).
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public Object readFieldWorker(BaseDataType var)
       throws EndConditionException,
              ErrorConditionException,
              StopConditionException
       {
          // load our buffer if it is empty
          if (currentIn == null)
          {
             currentIn = reload(false);

             // this only occurs in an unrecoverable error, silently return
             if (currentIn == null)
             {
                return null;
             }
          }

          // if a field is requested past the end of the current line buffer,
          // we are done
          if (indexIn >= currentIn.length())
          {
             // nothing to do
             return null;
          }

          // if delimiter is space then multiple sequential delimiters are ignored
          if (delim == DELIM)
          {
             while (indexIn < currentIn.length() && currentIn.charAt(indexIn) == delim)
             {
                // skip the extra delimiter
                indexIn++;

                // make sure we don't move past the end
                if (indexIn == currentIn.length())
                {
                   // nothing to do
                   return null;
                }
             }
          }

          String field = null;
          int    start = indexIn;
          int    end   = 0;

          // if the first character is a double quote, then we have to parse a
          // quoted string and handle escaped interior quotes (doubled up double
          // quotes)
          boolean asString = currentIn.charAt(start) == '\"';
          if (asString)
          {
             boolean force = false;

             // move our index past the initial quote
             indexIn++;

             // if we find our opening quote at the end of the input, force the
             // entry to the parsing loop below because we need to try to extend
             // our input buffer; this is more complicated because the loop
             // control structure of the parsing loop MUST drop out when the
             // index is incremented past the last character in the buffer EXCEPT
             // in this first iteration
             if (indexIn == currentIn.length())
             {
                force = true;
             }

             // find the end of the quoted string
             while (indexIn < currentIn.length() || force)
             {
                // shift back into normal operation
                force = false;

                // look for the next quote
                end = currentIn.indexOf('\"', indexIn);

                StringBuilder sb = new StringBuilder();
                int pos = -1;

                // there must always be a matching quote unless there is an
                // embedded newline
                while (end < 0)
                {
                   // keep reading until we find a closing quote or the end of
                   // the file, any data found is appended to our input buffer

                   // one time init
                   if (pos == -1)
                   {
                      pos = indexIn;
                      sb.append(currentIn);
                   }

                   // try to extend our line
                   try
                   {
                      String more = reload(true);

                      // this only occurs in an unrecoverable error, silently
                      // return
                      if (more == null)
                      {
                         return null;
                      }

                      // extend our data and add the embedded newline that was
                      // stripped off when we read the line in
                      sb.append(NEWLINE).append(more);

                      // update our state
                      currentIn = sb.toString();
                      end       = currentIn.indexOf('\"', pos);
                      pos       = currentIn.length();
                   }

                   catch (EndConditionException ece)
                   {
                      // no more data on this "line", continue processing here
                      // and suppress the ECE since it should not appear until
                      // the next call that tries to read a *new* line (we are
                      // trying to extend a current line which is treated
                      // differently in Progress); end will be a negative number
                      // already which is needed
                      break;
                   }
                }

                // check again for the matching quote
                if (end < 0)
                {
                   // we have read to the end of the file without a matching
                   // quote so we must drop all the data and return as if there
                   // is nothing left on this line

                   // increment to the end of the buffer
                   indexIn = currentIn.length();

                   // no assignment, just return (all data is ignored)
                   return null;
                }
                else
                {
                   // possible closing quote
                   if (++end < currentIn.length())
                   {
                      if (currentIn.charAt(end) == '\"')
                      {
                         // keep looping, this is just an escaped (doubled up)
                         // set of interior quotes
                         indexIn = end + 1;

                         // if we find our 2nd double quote character at the end
                         // of the input, force the parsing loop to iterate since
                         // we need to try to extend our input buffer; normally
                         // the parsing loop MUST drop out when the index is
                         // incremented past the last character in the buffer
                         // EXCEPT in this special case
                         if (indexIn == currentIn.length())
                         {
                            force = true;
                         }
                      }
                      else
                      {
                         // this is the end of the string and the end of the line
                         indexIn = end;

                         // exit the loop
                         break;
                      }
                   }
                   else
                   {
                      // this is the end of the string and the end of the line
                      indexIn = end;

                      // exit the loop
                      break;
                   }
                }
             }

             // read the field contents
             field = currentIn.substring(start, indexIn);

             // convert to an in-memory string (drop all the extra quotes)
             field = character.fromExportString(field);
          }
          else
          {
             // the non-quoted path (which can have embedded quotes so long as
             // they are not the first char after a delimiter)
             end = currentIn.indexOf(delim, start);

             if (end < 0)
             {
                // no delimiter is left, just copy the rest of the line by
                // setting the index to the length of the line
                indexIn = currentIn.length();
             }
             else
             {
                // we found the end of this field, increment the index
                indexIn = end;
             }

             // copy the data
             field = currentIn.substring(start, indexIn);
          }

          // increment the index past the next delimiter
          indexIn++;

          // implement 2 cases where we "skip" the field:
          // 1. null variable means the caller wants us to skip this field
          // 2. hyphen means skip this field (don't assign the var, just return)
          if (var == null || (field.equals("-") && skipLoneHyphenInput()))
          {
             // nothing to do
             return null;
          }

          // return the text of the read field
          return asString ? field : ("?".equals(field) ? UNKNOWN_VALUE : field);
       }

       /**
        * Read a complete line without regard to the normal Progress formatted field processing rules
        * or delimiters, attempting to import the data for the specified record fields.
        *
        * @param    entries
        *           The record to import into.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void readLine(FieldReference[] entries)
       {
          // only the first field gets read...
          if (entries.length == 0)
          {
             return;
          }

          readLine(entries[0]);
       }

       /**
        * Read a complete line without regard to the normal Progress formatted field processing rules
        * or delimiters, attempting to import the data for the specified record field.
        *
        * @param    entry
        *           The field to import into.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void readLine(FieldReference entry)
       {
          character line = new character();
          readLine(line);
          entry.set(line);
       }

       /**
        * Read a complete line without regard to the normal Progress formatted field processing rules
        * or delimiters, attempting to import the data for the specified legacy property.
        *
        * @param    entry
        *           The property to import into.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void readLine(PropertyReference entry)
       {
          character line = new character();
          readLine(line);
          entry.set(line);
       }

       /**
        * Move the index into the current line buffer past the next instance of
        * the specified delimiter using {@link #setDelimiter} or the default
        * delimiter of {@link #DELIM} and then return.
        * <p>
        * If the current line buffer is empty, a new line will be read from the
        * stream.
        * <p>
        * After all field reads for a given line are complete,
        * {@link #resetCurrentLine} must be called to enable input for the
        * next line.  Otherwise, all subsequent reads will continue trying to
        * read from the current line. This will result in reading data left on
        * the current line or reading nothing if the line has already been fully
        * read.  <code>resetCurrentLine</code> notifies the stream environment
        * that it should move to processing the next line.
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void skipField()
       throws EndConditionException,
              ErrorConditionException
       {
          // same thing as a readField() except we trash the return value
          readFieldWorker(null);
       }

       /**
        * Read a complete line without regard to the normal Progress formatted
        * field processing rules or delimiters.
        * <p>
        * At this time, this method is only called for implementing IMPORT UNFORMATTED.
        *
        * @param    var
        *           The <code>BaseDataType</code> variable to which the result is
        *           assigned.  This is contrary to Progress docs which state that
        *           the field must be a <code>character</code> field (the docs
        *           are incorrect).
        *
        * @throws   EndConditionException
        *           On EOF.
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void readLine(BaseDataType var)
       throws EndConditionException,
              ErrorConditionException,
              StopConditionException
       {
          if (protectReads())
          {
             return;
          }

          try
          {
             // for BinaryData targets, we need to read bytes and not lines
             String result = var instanceof BinaryData ? null : readNextLine(false, true);

             // check flag
             if (nullQuirk)
             {
                // no assignment case (happens when there is an embedded null
                // character in the data and there is no FOLLOWING line with
                // a newline before EOF)
                return;
             }

             if (var instanceof recid)
             {
                ((recid) var).assign(new recid(result));
             }
             else if (var instanceof integer)
             {
                // integer must come before int64 and date Otherwise the operator instanceof
                // will match the integer instances, too
                ((integer) var).assign(new integer(result));
             }
             else if (var instanceof int64)
             {
                ((int64) var).assign(new int64(result));
             }
             else if (var instanceof decimal)
             {
                ((decimal) var).assign(new decimal(result));
             }
             else if (var instanceof datetimetz)
             {
                // datetimetz must come before datetime and date Otherwise the operator instanceof
                // will match the datetimetz instances, too
                ((datetimetz) var).assign(new datetimetz(result));
             }
             else if (var instanceof datetime)
             {
                // datetime must come before date and date Otherwise the operator instanceof
                // will match the datetime instances, too
                ((datetime) var).assign(new datetime(result));
             }
             else if (var instanceof date)
             {
                ((date) var).assign(new date(result));
             }
             else if (var instanceof logical)
             {
                ((logical) var).assign(new logical(result));
             }
             else if (var instanceof character)
             {
                ((character) var).assign(new character(result));
             }
             else if (var instanceof clob)
             {
                ((clob) var).assign(new clob(result));
             }
             else if (var instanceof longchar)
             {
                ((longchar) var).assign(new longchar(result));
             }
             else if (var instanceof BinaryData)
             {
                // TODO: for process streams, 4GL 'ignores' next available line fully, and from next one to that,
                // the number of bytes as is the length of this binary data. Also, the raw variable gets set to
                // unknown always

                BinaryData bin = (BinaryData) var;
                int len = (int) bin.lengthOf();
                byte[] bytes = new byte[len];
                int nextPos = 0;

                outerLoop:
                while (true)
                {
                   byte[] nextBytes = readBytes((int) Math.min(BATCH_SIZE, len));
                   if (nextBytes == null)
                   {
                      if (nextPos == 0)
                      {
                         // if we are on EOF or otherwise error, raise an END
                         throw new EndConditionException("EOF");
                      }
                      break outerLoop;
                   }

                   for (int idx = 0; idx < nextBytes.length; idx++)
                   {
                      if (nextPos >= bytes.length)
                      {
                         break outerLoop;
                      }

                      byte nextByte = nextBytes[idx];
                      if (nextByte == 26 && isWindows && !binary)
                      {
                         setPosition(nextPos + 1);
                         // byte 26 (CTRL-Z) is interpreted as EOF only on Windows when in non-binary mode
                         break outerLoop;
                      }

                      // all '\r\n' sequences are replaced by '\n' (for '!binary && isWindows' case)
                      if (nextByte == '\n' && !binary && isWindows && nextPos > 0 && bytes[nextPos - 1] == '\r')
                      {
                         // check previous byte - if is '\r', replace it with '\n'
                         bytes[nextPos - 1] = '\n';
                      }
                      else
                      {
                         bytes[nextPos] = nextByte;
                         nextPos++;
                      }
                   }
                   len -= nextBytes.length;
                }

                // ensure only the actual read bytes are written
                if (nextPos != bytes.length)
                {
                   bytes = Arrays.copyOf(bytes, nextPos);
                }

                // set the byte content
                if (var instanceof raw)
                {
                   ((raw) var).assign(bytes);
                }
                else
                {
                   // do not overlay and do not trim the target LOB or MEMPTR buffer
                   ((LargeObject) var).write(false, bytes, 0, false);
                }
             }
             else if (var instanceof rowid)
             {
                ((rowid) var).assign(new rowid(result));
             }
          }
          catch (EOFException eof)
          {
             throw new EndConditionException("EOF");
          }
          catch (InterruptedIOException | InterruptedException iioe)
          {
             ErrorManager.handleStopException(new StopConditionException(iioe));
          }
          catch (IOException ioe)
          {
             // TODO: actual Progress compatible text and number are unknown
             ErrorManager.recordOrThrowError(-1, "IOException during read.");
          }
       }

       /**
        * Write a field to the stream using the previously specified delimiter
        * (using {@link #setDelimiter}) or the default delimiter of
        * {@link #DELIM} in between each field.  All output will be formatted
        * using the <code>toStringExport</code> method of the associated
        * type.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void writeField(int val)
       throws ErrorConditionException
       {
          writeField(new integer(val));
       }

       /**
        * Write a field to the stream using the previously specified delimiter
        * (using {@link #setDelimiter}) or the default delimiter of
        * {@link #DELIM} in between each field.  All output will be formatted
        * using the <code>toStringExport</code> method of the associated
        * type.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void writeField(double val)
       throws ErrorConditionException
       {
          writeField(new decimal(val));
       }

       /**
        * Write a field to the stream using the previously specified delimiter
        * (using {@link #setDelimiter}) or the default delimiter of
        * {@link #DELIM} in between each field.  All output will be formatted
        * using the <code>toStringExport</code> method of the associated
        * type.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void writeField(boolean val)
       throws ErrorConditionException
       {
          writeField(new logical(val));
       }

       /**
        * Write a field to the stream using the previously specified delimiter
        * (using {@link #setDelimiter}) or the default delimiter of
        * {@link #DELIM} in between each field.  All output will be formatted
        * using the <code>toStringExport</code> method of the associated
        * type.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void writeField(String val)
       throws ErrorConditionException
       {
          writeField(new character(val));
       }

       /**
        * Write a field to the stream using the previously specified delimiter
        * (using {@link #setDelimiter}) or the default delimiter of
        * {@link #DELIM} in between each field.  All output will be formatted
        * using the <code>toStringExport</code> method of the associated
        * type.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void writeField(BaseDataType val)
       throws ErrorConditionException
       {
          // are we the first field in this output line?
          if (column > 0)
          {
             // not the first, so we write the delimiter
             append(delim);
          }

          // write the actual field in export "format"
          append(val.toStringExport());
       }

       /**
        * Write the entire block of data (as represented by the given  <code>BinaryData</code>) to the stream
        * with no delimiters, end of line insertions or other modifications.  All output will not be formatted
        * and exported using the <code>getByteArray</code> method of the associated type.
        * <p>
        * Use of this method does NOT require bracketing between
        * {@link #startOutput} and {@link #endOutput} methods and these must
        * not be called otherwise the result is undefined.
        * <p>
        * This method is called only from client-side, via the ExportField.output method.
        * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void writeBlock(BinaryData val)
       throws ErrorConditionException,
              StopConditionException
       {
          // TODO: memptr may be on server-side, improve this

          try
          {
             if (val.isUnknown())
             {
                write("?");
             }
             else
             {
                if (val.lengthOf() > 0)
                {
                   // negative array sizes cause a memory access violation in 4GL
                   // but here we just avoid the failure

                   // TODO: test zero length array sizes

                   write(val.getByteArray());
                }
             }
          }

          catch (InterruptedIOException iioe)
          {
             ErrorManager.handleStopException(new StopConditionException(iioe));
          }

          catch (IOException ioe)
          {
             ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
          }
       }

       /**
        * Write the entire block of data (as represented by the given <code>blob</code>) to the stream with no
        * delimiters, end of line insertions or other modifications.  The data will be exported into separate
        * file within subdirectory defined in LOB-DIR option or if not defined in current base directory.
        * <p>
        * Use of this method does NOT require bracketing between {@link #startOutput} and {@link #endOutput}
        * methods and these must not be called otherwise the result is undefined.
        * <p>
        * This method is called only from client-side, via the ExportField.output method.
        * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void writeBlob(blob val)
       throws ErrorConditionException,
              StopConditionException
       {
          writeBlock(val);
       }

       /**
        * Write the entire block of data (as represented by the given <code>clob</code>) to the stream with no
        * delimiters, end of line insertions or other modifications.  The data will be exported into separate
        * file within subdirectory defined in LOB-DIR option or if not defined in current base directory.
        * <p>
        * Use of this method does NOT require bracketing between {@link #startOutput} and {@link #endOutput}
        * methods and these must not be called otherwise the result is undefined.
        * <p>
        * This method is called only from client-side, via the ExportField.output method.
        * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
        *
        * @param    val
        *           The value to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void writeClob(clob val)
       throws ErrorConditionException,
              StopConditionException
       {
          writeField(val);
       }

       /**
        * Inserts a page break if the stream is paged and if the next write
        * position is not on the first line of a new page.  This method
        * internally implements a {@link #startOutput} and {@link #endOutput}.
        * This means that this method should never be called while there is
        * a current <code>startOutput</code> is in effect.
        * <p>
        * If page footers are in use, any partial page will be extended to the
        * full size before the footers are output.
        * <p>
        * This method does nothing if the stream is in <code>control</code> mode.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void advancePage()
       throws ErrorConditionException
       {
          if (protectWrites())
             return;

          if (!control && rawGetPageSize() > 0)
          {
             // TODO: it is unknown if an explicit NL should be appended here
             endOutput();
             processAdvanceNotifications();

             if (rawGetNextLineNum() > 1)
             {
                startOutput(0);

                try
                {
                   extendPage();
                }
                catch (IOException ioe)
                {
                   ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
                }

                writePageBreak();
                endOutput();
             }
          }
       }

       /**
        * This method does nothing if the stream is in <code>control</code> mode.
        */
       public void flush()
       {
          if (flushListeners != null)
          {
             Iterator<FlushListener> list = flushListeners.iterator();

             while (list.hasNext())
             {
                FlushListener fl = list.next();
                fl.notifyFlush(this);
                list.remove();
             }
          }
       }

       /**
        * Create the buffer to gather all output of {@link #putField},
        * {@link #putSpace}, {@link #putLineEnd} and {@link #writeField} methods.
        * When filled, the buffer must be written to the stream using
        * {@link #endOutput}.
        * <p>
        * There must always be an intervening <code>endOutput</code> between
        * every two calls to this method.  Do not try to nest calls to this
        * method.
        * <p>
        * This method does nothing if the stream is in <code>control</code> mode.
        *
        * @param    newColumn
        *           Initial value for column counter.
        */
       public void startOutput(int newColumn)
       {
          processHeaders();

          if (!control)
          {
             column = newColumn;

             if (active)
                return;

             currentOut = new StringBuilder();
             active     = true;
          }
       }

       /**
        * Write the current output buffer to the stream. This flushes all
        * buffered output of the {@link #putField}, {@link #putSpace},
        * {@link #putLineEnd} and {@link #writeField} methods to the stream.
        * This method also resets the current delimiter to the default.
        * <p>
        * This method does nothing if the stream is in <code>control</code> mode.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void endOutput()
       throws ErrorConditionException,
              StopConditionException
       {
          endOutput(false);
       }

       /**
        * Write the current output buffer to the stream. This flushes all
        * buffered output of the {@link #putField}, {@link #putSpace},
        * {@link #putLineEnd} and {@link #writeField} methods to the stream.
        * This method also resets the current delimiter to the default.
        * <p>
        * This method does nothing if the stream is in <code>control</code> mode.
        *
        * @param    appendNewLine
        *           Automatically add new line at the end of output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void endOutput(boolean appendNewLine)
       throws ErrorConditionException,
              StopConditionException
       {
          if (!control)
          {
             if (!active)
                return;

             try
             {
                if (!trailingNL && appendNewLine)
                {
                   writeNewLine();
                   rawIncrementNextLineNum();
                   setTrailingNL(true);
                }

                String output = currentOut.toString();

                write(output);

                // we will count only if the written char is not a page-break
                if (!"\f".equals(output))
                   lastPageWithOutput = rawGetPageNum();
             }

             catch (InterruptedIOException iioe)
             {
                ErrorManager.handleStopException(new StopConditionException(iioe));
             }

             catch (IOException ioe)
             {
                ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
             }

             finally
             {
                // if the put worker still in progress we need to just clean the buffer instead of
                // removing it leaving them active and ready for incoming new data
                synchronized (outActivityLock)
                {
                   if (putWorkerActive)
                   {
                      currentOut.setLength(0);
                   }
                   else
                   {
                      currentOut = null;
                      active     = false;
                   }
                }
             }
          }
       }

       /**
        * Notifies the stream that an empty drawing has occurred.
        * This has the effect of kicking off the page headers if at the page
        * boundary.
        */
       public void pulseOutput()
       {
          if (!control && rawGetPageSize() > 0 && rawGetNextLineNum() == 1)
          {
             processHeaders();
          }
       }

       /**
        * Append a platform-specific line-end to the current output buffer if
        * the output is not already positioned at the beginning of a line OR
        * if in <code>control</code> mode.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        * <p>
        * This method is built to duplicate the behavior of the SKIP option with
        * the put statement.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putLineEnd()
       throws ErrorConditionException
       {
          putLineEnd(0);
       }

       /**
        * Append the specified number of platform-specific line-ends to the
        * current output buffer.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param   num
        *          The number of blank lines to follow the current line (which
        *          means that this number of line-ends + 1 must be appended).
        *          If this is negative, no output is generated. If 0, 1 line-end
        *          is output if (and only if) the current output position is not
        *          already at the beginning of the line.  If 1, then 2 line-ends
        *          are output and so on.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putLineEnd(int num)
       throws ErrorConditionException
       {
          if (num < 0)
          {
             // ignore the call
             return;
          }
          else
          {
             // compute the number of remaining empty rows
             // remaining-empty-rows = PAGE-SIZE - LINE-COUNTER + 1 - footerSize
             int pageSz = rawGetPageSize();
             int nextLineNum = rawGetNextLineNum();
             int remainingRows = pageSz - nextLineNum  + 1 - footerSize;

             // do not adjust number of rows when page size is not set
             if (pageSz > 0 && num > 0 && num > remainingRows)
             {
                num = remainingRows;
             }

             // add NL if there was unfinished output
             processLineEnd(true, true);
             for (int i = 0; i < num; i++)
             {
                processLineEnd(false, true);
             }
          }
       }

       /**
        * Append a space to the current output buffer.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putSpace()
       throws ErrorConditionException
       {
          append(' ');
       }

       /**
        * Append the given number of spaces to the current output buffer.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    num
        *           The number of spaces to append.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putSpace(int num)
       throws ErrorConditionException
       {
          for (int i = 0; i < num; i++)
          {
             append(' ');
          }
       }

       /**
        * Append a null character '\0' to the current output buffer.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putNull()
       throws ErrorConditionException
       {
          append('\0');
       }

       /**
        * Append the given number of null characters '\0' to the current output
        * buffer.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    num
        *           The number of nulls to append.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putNull(int num)
       throws ErrorConditionException
       {
          for (int i = 0; i < num; i++)
          {
             append('\0');
          }
       }

       /**
        * Append the <code>int</code> value to the output buffer with the default
        * format string of '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(int val)
       throws ErrorConditionException
       {
          putField(val, null);
       }

       /**
        * Append the <code>int</code> value to the output buffer with the given
        * format string at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' will be used.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(int val, String fmt)
       throws ErrorConditionException
       {
          putField(new integer(val), fmt);
       }

       /**
        * Append the <code>int</code> value to the output buffer with the default
        * format string of '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(int val, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col);
       }

       /**
        * Append the <code>int</code> value to the output buffer with the given
        * format string at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(int val, String fmt, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(new integer(val), fmt, starts, col);
       }

       /**
        * Append the <code>int</code> value to the output buffer with the default
        * format string of '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(int val, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col.intValue());
       }

       /**
        * Append the <code>int</code> value to the output buffer with the given
        * format string at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(int val, String fmt, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(new integer(val), fmt, starts, col.intValue());
       }

       /**
        * Append the <code>double</code> value to the output buffer with the
        * default format string of '-&gt;&gt;,&gt;&gt;9.99' at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(double val)
       throws ErrorConditionException
       {
          putField(val, null);
       }

       /**
        * Append the <code>double</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           '-&gt;&gt;,&gt;&gt;9.99' will be used.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(double val, String fmt)
       throws ErrorConditionException
       {
          putField(new decimal(val), fmt);
       }

       /**
        * Append the <code>double</code> value to the output buffer with the
        * default format string of '-&gt;&gt;,&gt;&gt;9.99' at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(double val, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col);
       }

       /**
        * Append the <code>double</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           '-&gt;&gt;,&gt;&gt;9.99' will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(double val, String fmt, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(new decimal(val), fmt, starts, col);
       }

       /**
        * Append the <code>double</code> value to the output buffer with the
        * default format string of '-&gt;&gt;,&gt;&gt;9.99' at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(double val, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col.intValue());
       }

       /**
        * Append the <code>double</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Note that this format string works as the display form, which pads on
        * the left to the full size of the format string.  This is different from
        * the normal operation of <code>toString</code>.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           '-&gt;&gt;,&gt;&gt;9.99' will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(double     val,
                            String     fmt,
                            boolean    starts,
                            NumberType col)
       throws ErrorConditionException
       {
          putField(new decimal(val), fmt, starts, col.intValue());
       }

       /**
        * Append the <code>boolean</code> value to the output buffer with the
        * default format string of 'yes/no' at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(boolean val)
       throws ErrorConditionException
       {
          putField(val, null);
       }

       /**
        * Append the <code>boolean</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           'yes/no' will be used.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(boolean val, String fmt)
       throws ErrorConditionException
       {
          putField(new logical(val), fmt);
       }

       /**
        * Append the <code>boolean</code> value to the output buffer with the
        * default format string of 'yes/no' at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(boolean val, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col);
       }

       /**
        * Append the <code>boolean</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           'yes/no' will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(boolean val, String fmt, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(new logical(val), fmt, starts, col);
       }

       /**
        * Append the <code>boolean</code> value to the output buffer with the
        * default format string of 'yes/no' at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(boolean val, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col.intValue());
       }

       /**
        * Append the <code>boolean</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           'yes/no' will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(boolean    val,
                            String     fmt,
                            boolean    starts,
                            NumberType col)
       throws ErrorConditionException
       {
          putField(new logical(val), fmt, starts, col.intValue());
       }

       /**
        * Append the <code>String</code> value to the output buffer with a
        * format string of 'x(N)' (where N is the length of the value) at the
        * current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(String val)
       throws ErrorConditionException
       {
          putField(val, null);
       }

       /**
        * Append the <code>String</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           'x(N)' will be used (where N is the length of the value).
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(String val, String fmt)
       throws ErrorConditionException
       {
          putField(new character(val), generateStringFormat(fmt, val.length()));
       }

       /**
        * Append the <code>String</code> value to the output buffer with the
        * default format string of 'x(N)' will be used (where N is the length of
        * the value) at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(String val, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col);
       }

       /**
        * Append the <code>String</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           'x(N)' will be used (where N is the length of the value) will
        *           be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(String val, String fmt, boolean starts, int col)
       throws ErrorConditionException
       {
          fmt = generateStringFormat(fmt, val.length());
          putField(new character(val), fmt, starts, col);
       }

       /**
        * Append the <code>String</code> value to the output buffer with the
        * default format string of 'x(N)' will be used (where N is the length of
        * the value) at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(String val, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col.intValue());
       }

       /**
        * Append the <code>String</code> value to the output buffer with the
        * given format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           'x(N)' will be used (where N is the length of the value) will
        *           be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(String     val,
                            String     fmt,
                            boolean    starts,
                            NumberType col)
       throws ErrorConditionException
       {
          fmt = generateStringFormat(fmt, val.length());
          putField(new character(val), fmt, starts, col.intValue());
       }

       /**
        * Append the value to the output buffer with the default type-specific
        * format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(BaseDataType val)
       throws ErrorConditionException
       {
          putField(val, null);
       }

       /**
        * Append the value to the output buffer with the given type-specific
        * format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           type-specific format will be used.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(BaseDataType val, String fmt)
       throws ErrorConditionException
       {
          if (control && val instanceof raw)
          {
             try
             {
                write(((raw)val).getByteArray());
             }

             catch (InterruptedIOException iioe)
             {
                ErrorManager.handleStopException(new StopConditionException(iioe));
             }

             catch (IOException ioe)
             {
                ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
             }
             return;
          }
          append(generateFormattedText(val, fmt));
       }

       /**
        * Append the value to the output buffer with the default type-specific
        * format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(BaseDataType val, boolean starts, int col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col);
       }

       /**
        * Append the value to the output buffer with the default type-specific
        * format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(BaseDataType val, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(val, null, starts, col.intValue());
       }

       /**
        * Append the value to the output buffer with the given type-specific
        * format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           type-specific format will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(BaseDataType val, String fmt, boolean starts, NumberType col)
       throws ErrorConditionException
       {
          putField(val, fmt, starts, col.intValue());
       }

       /**
        * Append the value to the output buffer with the given type-specific
        * format string at the current output column.
        * <p>
        * Use of this method generates output to a buffer which must first be
        * initialized by {@link #startOutput}.  Once initialized, any number of
        * calls to this method (or related output methods) may be done and all
        * output will be gathered into the output buffer.  To write this buffer
        * to the stream, call {@link #endOutput}. Calls to this method must never
        * be done outside of a bracketed pair of matching <code>startOutput</code>
        * and <code>endOutput</code> calls otherwise the results will be
        * undefined.
        *
        * @param    val
        *           The data to be output.
        * @param    fmt
        *           The format string to use.  If <code>null</code>, the default
        *           type-specific format will be used.
        * @param    starts
        *           If <code>true</code>, the output will be placed starting at
        *           the given column (or on the next line at that column, if the
        *           given column is already in use on the current line). If
        *           <code>false</code>, the output will be placed so that it
        *           ends at the given column (or on the next line at that column,
        *           if the calculated start column is already in use on the
        *           current line).
        * @param    col
        *           The start or end column (which it is depends on the previous
        *           parameter's value). This is used as a 1-based index into the
        *           current output line.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putField(BaseDataType val, String fmt, boolean starts, int col)
       throws ErrorConditionException
       {
          // obtain the text to output, based on type-specific formatting
          String text = generateFormattedText(val, fmt);
          int    len  = text.length();

          // convert the 1-based index into the 0-based index that we need
          col--;

          // calculate starting position if needed
          if (!starts)
          {
             col = (col + 1) - len;
          }

          // do we need to move to the next line?
          if (col < column)
          {
             processLineEnd(false, true);

             // we must check for page-break after we output this NL
             checkForPageBreak();
          }

          int pad = col - column;

          // pad if necessary
          if (pad > 0)
          {
             putSpace(pad);
          }

          // output the text
          append(text);
       }

       /**
        * Output a list of fields (without buffering) to the stream.
        *
        * @param    data
        *           The list of fields to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putControl(FieldEntry[] data)
       throws ErrorConditionException
       {
          if (protectWrites())
          {
             return;
          }

          endOutput();

          // this sets this value on the server side (in case it is needed)
          setControl(true);

          try
          {
             putWorker(data, MODE_PUT_CONTROL, (char) -1);
          }
          finally
          {
             // this resets this value on the server side
             setControl(false);
          }
       }

       /**
        * Output a list of fields with default formatting to the stream.
        *
        * @param    data
        *           The list of fields to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putUnformatted(FieldEntry[] data)
       throws ErrorConditionException
       {
          if (protectWrites())
          {
             return;
          }

          putWorker(data, MODE_PUT_UNFORMATTED, (char) -1);
       }

       /**
        * Output a list of fields with specified formatting to the stream.
        *
        * @param    data
        *           The list of fields to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void put(FieldEntry[] data)
       throws ErrorConditionException
       {
          if (protectWrites())
          {
             return;
          }

          // by default no automatic line feed should be performed
          putWorker(data, MODE_PUT_NORMAL, (char) -1);
       }

       /**
        * Output a list of fields with export formatting to the stream.
        *
        * @param    data
        *           The list of fields to be output.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        * @throws   StopConditionException
        *           If any interruption occurs during I/O processing.
        */
       public void export(FieldEntry[] data)
       throws ErrorConditionException,
              StopConditionException
       {
          if (protectWrites())
          {
             return;
          }

          putWorker(data, MODE_EXPORT, delim);
       }

       /**
        * Writes the given data into a buffer appending a newline to the end.
        * This buffered data will be pushed out to the stream with a subsequent
        * call to {@link #flushControlled}.
        *
        * @param    txt
        *           The data to write.  May be <code>null</code> in which case
        *           nothing is written to the buffer. If this is the empty
        *           string, only a newline is written into the buffer.
        */
       public void writeControlled(String txt)
       {
          boolean isNeedNL = column != 0 && !inHeaders && !trailingWithoutPutNL;

          if (txt == null)
          {
             //nothing to do
             return;
          }

          if (buffer == null)
          {
             buffer = new ArrayList<>();
          }
          StringBuilder sb = new StringBuilder(txt.length() + 2);
          if (isNeedNL)
          {
             trailingWithoutPutNL = true;
             sb.append(NEWLINE);
          }
          sb.append(txt).append(NEWLINE);

          buffer.add(sb.toString());
       }

       /**
        * Supports writing form feeds and intermediate headers.
        * Required for a proper implementation of redirected editors.
        * Writes the given data into a buffer and immediately flushes the buffer
        * to the stream. Advances the page number and adjusts the line number
        * depending on the height of the headers. Suppresses the form feed output
        * for the nearest page break.
        *
        * @param    txt
        *           The data to write. The form feed is expected to be the first
        *           character, followed by optional frame header lines, although
        *           it is not checked
        * @param    lines
        *           number of header lines in the passed text. required for
        *           adjusting the line counter
        */
       public void writeControlledSpecial(String txt, int lines)
       {
          if (buffer == null)
          {
             buffer = new ArrayList<>();
          }

          buffer.add(txt);

          boolean savecont = contiguous;
          contiguous = true;
          try
          {
             flushControlled(true);
          }
          catch (IOException ioe)
          {
          }
          contiguous = savecont;

          // maintain paging state
          rawIncrementPageNum();
          rawResetNextLineNum();
          for (int i = 0; i < lines; i++)
          {
             rawIncrementNextLineNum();
          }

          setTrailingNL(true);
          suppressFormFeed = true;
       }

       /**
        * Writes the data previously buffered (via {@link #writeControlled}) to
        * the stream.  This method maintains all needed state for proper paging
        * processing and it will output report headers or footers as needed.
        *
        * @param    bypass
        *           <code>true</code> if header processing must be bypassed when
        *           starting to flush the output.
        *
        * @return   The next line number AFTER all buffered lines are output but
        *           BEFORE any final page break processing has occurred. This can
        *           be compared with the next line number as queried after this
        *           method returns to determine if a page break was output or
        *           not (among other uses). If there was no output, then -1 will
        *           be returned.
        *
        * @throws   IOException
        *           if an I/O error occurs.
        */
       public int flushControlled(boolean bypass)
       throws IOException
       {
          int rc = -1;

          if (buffer == null || buffer.size() == 0)
          {
             //nothing to do
             return rc;
          }

          ArrayList<String> buf  = buffer;
          Iterator<String>  list = buffer.iterator();
          flushLevel++;

          // "clear" the buffer so that we can recursively call in to fill the buffer again
          buffer = null;

          try
          {
             // only output if this is the top-level entry to this method (this
             // method has a form of indirect recursion)
             if (flushLevel == 1)
             {
                // first flush the internal buffer if any
                endOutput();

                // if we are NOT on the first line of the page AND the expected
                // output will cross a page boundary AND we are not in contiguous
                // mode then handle the form feed, page state and footers now
                // (causes recursion)

                // WARNING: This protection is needed in the case where we are
                //          outputting exactly as many lines as there are
                //          left on the page.  Progress does NOT trigger the
                //          page break in that scenario.

                int top            = rawGetNextLineNum();
                int sz             = buf.size();
                int remainingLines = 1;
                int pageSz         = rawGetPageSize();

                if (pageSz > 0)
                {
                   remainingLines = pageSz - top + 1 - footerSize;
                }

                if (!doneOut             &&
                    top > 1              &&
                    sz  > 1              &&
                    sz  > remainingLines &&
                    !contiguous          &&
                    needsPageBreak(sz))
                {
                   // fill in a partial page with blanks before processing any footer or form feed
                   extendPage();
                   writePageBreak();

                   // if the page-break was written, then page headers must be output
                   bypass = false;
                }

                if (!bypass)
                {
                   // if we are at the beginning of the page, output the headers (causes recursion)
                   processHeaders();
                }
             }

             // process our buffered output (strangely, Progress always groups the
             // entire buffered output together, even if it will cause a longer or
             // shorter page size than specified in the stream definition --> this
             // is why the header/footer processing is outside of this loop)
             while (list.hasNext())
             {
                String line = list.next();

                write(line);
                rawIncrementNextLineNum();
                setTrailingNL(true);
             }

             // get return value after the buffer is processed but before the
             // possible ending page break
             rc = rawGetNextLineNum();

             // if we are AT or PAST the end of the page, handle the form feed,
             // page state and footers (causes recursion)
             if (flushLevel == 1 && !doneOut && !contiguous && needsPageBreak())
             {
                writePageBreak();
             }
          }
          finally
          {
             if (flushLevel == 1)
             {
                contiguous = false;
             }

             flushLevel--;
          }

          return rc;
       }

       /**
        * Detects if headers need to be output.
        *
        * @return   <code>true</code> if header processing is not in process and
        *           the stream has registered page headers and the next line
        *           number is 1 (we have no output on the current page).
        */
       public boolean needsHeadersOutput()
       {
          return (!inHeaders && headers != null && rawGetNextLineNum() == 1);
       }

       /**
        * Detects if output is being generated on a new page and renders the
        * header(s) if needed.
        */
       public void processHeaders()
       {
          // avoid recursion
          if (inHeaders)
             return;

          // if starting output on a new page
          if (needsHeadersOutput())
          {
             processHeadersInt();
          }
       }

       /**
        * Detects if output is being generated on a new page and renders the
        * header(s) if needed.
        */
       public void processHeadersUncond()
       {
          try
          {
             flushControlled(true);
          }
          catch (IOException ioe)
          {
          }

          processHeadersInt();

          suppressFormFeed = false;
       }

       /**
        * Detects if output is being generated on a new page and renders the
        * header(s) if needed.
        */
       public void processHeadersInt()
       {
          inHeaders = true;

          boolean old = false;

          if (flushLevel == 0)
          {
             // preserve some flags that will possibly be reset after the call
             // to render headers since flushing resets the flag (not to
             // mention that we force the flag value too)
             old = contiguous;

             // we need contiguous mode to avoid triggering a page break in
             // the end of flushing since page sizes that are smaller than the
             // header size will detect that a page break is needed
             contiguous = true;
          }

          // output headers
          processRendering(headers, false);

          // restore previous flag value
          if (flushLevel == 0)
          {
             contiguous = old;
          }

          inHeaders = false;
       }

       /**
        * Output any registered footers in a LIFO order.
        */
       public void processFooters()
       {
          Set<RepeatedOutput> out = footers;

          // add the footers in a LIFO order.
          // TODO: performance could be improved by removing intermediary ArrayList container
          if (out.size() > 1)
          {
             List<RepeatedOutput> l = new ArrayList<>();

             for (RepeatedOutput anOut : out)
             {
                l.add(0, anOut);
             }

             out = new LinkedHashSet<>(l);
          }

          processRendering(out, true);
       }

       /**
        * Checks if this is a paged stream and that it has at least 1 header or
        * footer registered.
        *
        * @return   <code>true</code> if this is a paged stream with at least
        *           1 header or footer.
        */
       public boolean hasPageHeadersFooters()
       {
          boolean hdrs = (headers != null && !headers.isEmpty());
          boolean ftrs = (footers != null && !footers.isEmpty());

          return (rawGetPageSize() > 0 && (hdrs || ftrs));
       }

       /**
        * Adds a listener to the list of those objects that will get a
        * notification at the top of every new page (except for the first
        * page).
        *
        * @param    pl
        *           The object to receive a notification whenever a page break is
        *           written to this stream.
        */
       public void registerPageBreakListener(PageBreakListener pl)
       {
          if (pageListeners == null)
          {
             pageListeners = new WeakHashMap<>();
          }

          pageListeners.put(pl, null);
       }

       /**
        * Adds a listener to the list of those objects that will get a
        * notification when this stream is about to be closed.
        *
        * @param    cl
        *           The object to receive a notification
        */
       public void registerCloseListener(CloseListener cl)
       {
          if (closeListeners == null)
          {
             closeListeners = new WeakHashMap<>();
          }

          closeListeners.put(cl, null);
       }

       /**
        * Adds a listener to the list of those objects that will get a
        * notification when this stream is about to be flushed.
        *
        * @param    fl
        *           The object to receive a notification
        */
       public void registerFlushListener(FlushListener fl)
       {
          if (flushListeners == null)
          {
             flushListeners = new LinkedHashSet<>();
          }

          flushListeners.add(fl);
       }

       /**
        * Adds a listener to the list of those objects that will get a
        * notification when this stream gets page advance order.
        *
        * @param    al
        *           The object to receive a notification
        */
       public void registerAdvanceListener(AdvanceListener al)
       {
          if (advanceListeners == null)
          {
             advanceListeners = new LinkedHashSet<>();
          }

          advanceListeners.add(al);
       }

       /**
        * Adds a header to the list of those headers that will be output at the
        * top of every new page.
        *
        * @param    head
        *           The header to add.
        */
       public void registerHeader(RepeatedOutput head)
       {
          if (rawGetPageSize() == 0)
          {
             // not paged so no headers are needed
             return;
          }

          if (headers == null)
          {
             headers = new LinkedHashSet<>();
          }

          if (headers.add(head))
          {
             headerSize += head.height();
          }
       }

       /**
        * Adds a footer to the list of those footers that will be output at the
        * bottom of every page (even a last partial page).
        *
        * @param    foot
        *           The footer to add.
        */
       public void registerFooter(RepeatedOutput foot)
       {
          if (rawGetPageSize() == 0)
          {
             // not paged so no footers are needed
             return;
          }

          if (footers == null)
          {
             footers = new LinkedHashSet<>();
          }

          if (footers.add(foot))
          {
             footerSize += foot.height();
          }
       }

       /**
        * Removes a header from the list of those headers that will be output at the
        * top of every new page.
        *
        * @param    head
        *           The header to remove.
        */
       public void deregisterHeader(RepeatedOutput head)
       {
          if (headers == null)
          {
             return;
          }

          if (headers.remove(head))
          {
             headerSize -= head.height();
          }

          if (headers.isEmpty())
          {
             headers = null;
             headerSize = 0;
          }
       }

       /**
        * Removes a footer from the list of those footers that will be output at
        * the bottom of every page (even a last partial page).
        *
        * @param    foot
        *           The footer to remove.
        */
       public void deregisterFooter(RepeatedOutput foot)
       {
          if (footers == null)
          {
             return;
          }

          if (footers.remove(foot))
          {
             footerSize -= foot.height();
          }

          if (footers.isEmpty())
          {
             footers = null;
             footerSize = 0;
          }
       }

       /**
        * Obtains the current read and/or write position in the stream, if the stream type supports
        * such a query.
        * <p>
        * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They
        * are mapped to the underlying Java platform that can handle streams with offsets that are
        * based on a 64-bit <code>long</code>.
        *
        * @return  The current offset into the stream or the <code>unknown value</code> if the
        *          stream type doesn't support a stream position.
        */
       public int64 getPosition()
       {
          long pos = 0;

          try
          {
             pos = getPos();
          }
          catch (Exception exc)
          {
             // return the unknown value because the stream doesn't support querying the seek
             // position (UnsupportedOperationException) or there was an IOException
             pos = -1;
          }

          // return the queried position
          return (pos == -1) ? new int64() : new int64(pos);
       }

       /**
        * Sets the current read and/or write position in the stream, if the stream type supports the
        * operation.
        * <p>
        * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They
        * are mapped to the underlying Java platform that can handle streams with offsets that are
        * based on a 64-bit <code>long</code>.
        * <p>
        * A negative offset is ignored (no action is taken).
        * <p>
        * If the specified offset is larger than the length of the stream, in
        * Progress the seek position will report as the number requested, but if
        * no subsequent writes occur to the file, the file size is left the same
        * as it was before setting the position.  If writes do occur, all writes
        * occur at the new position. In this case, the resulting file will be
        * sized based on the requested offset + the number of bytes written.
        *
        * @param    newpos
        *           The new offset.  If this is the <code>unknown value</code>,
        *           the set operation will silently return without any change.
        */
       public void setPosition(int64 newpos)
       {
          if (newpos.isUnknown())
          {
             // nothing to do
             return;
          }

          setPosition(newpos.longValue());
       }

       /**
        * Sets the current read and/or write position in the stream, if the
        * stream type supports the operation.
        * <p>
        * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They
        * are mapped to the underlying Java platform that can handle streams with offsets that are
        * based on a 64-bit <code>long</code>.
        * <p>
        * A negative offset is ignored (no action is taken).
        * <p>
        * If the specified offset is larger than the length of the stream, in
        * Progress the seek position will report as the number requested, but if
        * no subsequent writes occur to the file, the file size is left the same
        * as it was before setting the position.  If writes do occur, all writes
        * occur at the new position. In this case, the resulting file will be
        * sized based on the requested offset + the number of bytes written.
        *
        * @param    newpos
        *           The new offset.  If this is the <code>unknown value</code>,
        *           the set operation will silently return without any change.
        */
       public void setPosition(long newpos)
       {
          if (newpos < 0)
          {
             // nothing to do
             return;
          }

          try
          {
             setPos(newpos);
          }
          catch (Exception exc)
          {
             // silently fail on UnsupportedOperationException or IOException
          }
       }

       /**
        * Set the current stream's read/write position to the end of the file
        * such that future writes are appending to the file.
        */
       public void setAppend()
       {
          try
          {
             setPos(getLen());
          }
          catch (Exception exc)
          {
             // silently fail on UnsupportedOperationException or IOException
          }
       }

       /**
        * The length of the stream in bytes.  This does not work for "streams"
        * that require sequential access.
        *
        * @return   The length of the stream or the <code>unknown value</code>
        *           if the underlying stream does not support this operation.
        */
       public int64 getLength()
       {
          long len = 0;

          try
          {
             len = getLen();
          }
          catch (Exception exc)
          {
             // return the unknown value because the stream doesn't support querying length
             // (UnsupportedOperationException) or there was an IOException
             return new int64();
          }

          // return the queried length
          return new int64(len);
       }

       /**
        * Processes the notification that the scope in which the stream was
        * instantiated is ending and the object's reference will be lost after
        * this method is called.  This provides a natural and standard mechanism
        * to cleanup any resources that may need to be closed explicitly such
        * as operating system files or sockets.
        */
       public void finished()
       {
          // flush collected data if any
          endOutput();
          close();
       }

       /**
        * Provides a notification that the external program scope in which the object is registered is
        * is being deleted and the object's reference will be lost after this method is called.
        * <p>
        * This is a no-op for streams.
        */
       @Override
       public void deleted()
       {
          // no-op
       }

       /**
        * Processes the iterate notification provided by the {@link
        * TransactionManager}.  This implementation does nothing.
        */
       public void iterate()
       {
       }

       /**
        * Processes the retry notification provided by the {@link
        * TransactionManager}.  This implementation does nothing.
        */
       public void retry()
       {
       }

       /**
        * Generate the default format string of 'x(N)' will be used (where N is
        * the length of the value) IF AND ONLY IF the given input format string
        * is <code>null</code>.
        *
        * @param    fmt
        *           The format string (return unchanged if not <code>null</code>)
        *           of <code>null</code> to trigger the generation of a default
        *           string-length format.
        * @param    len
        *           Fixed string value length.
        *
        * @return   The input format string or a default string-length format.
        */
       public static String generateStringFormat(String fmt, int len)
       {
          if (fmt == null)
          {
             fmt = "x(" + len + ')';
          }

          return fmt;
       }

       /**
        * The number of bytes available to be immediately read without blocking.
        *
        * @return   The number of available bytes.
        *
        * @throws   IOException
        *           if an I/O error occurs.
        */
       public abstract long available()
       throws IOException;

       /**
        * The 0-based offset into the stream at which the next read or write will
        * occur.  This does not work for "streams" that require sequential
        * access.
        *
        * @return   The current position in the stream.
        *
        * @throws   UnsupportedOperationException
        *           If the requested operation is not supported.
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract long getPos()
       throws UnsupportedOperationException,
              IOException;

       /**
        * Moves the current read/write position to the specified absolute 0-based
        * offset.  A negative offset is ignored (no action is taken).  If the
        * specified offset is larger than the length of the stream, the seek
        * position will report as the number requested, but if no subsequent
        * writes occur to the file, the file is truncated to 0 bytes.  If writes
        * do occur, all writes occur at byte 0 BUT the file actually is of a size
        * that is the requested offset + the number of bytes written!
        * <p>
        * This does not work for "streams" that require sequential access.
        *
        * @param    pos
        *           The new read/write position in the stream.
        *
        * @throws   UnsupportedOperationException
        *           If the requested operation is not supported.
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract void setPos(long pos)
       throws UnsupportedOperationException,
              IOException;

       /**
        * The length of the stream in bytes.  This does not work for "streams"
        * that require sequential access.
        *
        * @return   The length of the stream.
        *
        * @throws   UnsupportedOperationException
        *           If the requested operation is not supported.
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract long getLen()
       throws UnsupportedOperationException,
              IOException;

       /**
        * Truncates or extends the stream to the specified length if this stream
        * supports such an operation. If truncation is requested, all data
        * located after this point in the file is discarded.  If extending the
        * file is requested, the values of the data in the extended portion of
        * the file is undefined.
        *
        * @param    len
        *           The new length of the file.
        *
        * @throws   UnsupportedOperationException
        *           If the requested operation is not supported.
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract void setLen(long len)
       throws UnsupportedOperationException,
              IOException;

       /**
        * Write the given character to the output stream.
        *
        * @param    ch
        *           The character to be written.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract void writeCh(char ch)
       throws IOException;

       /**
        * Write the given byte to the output stream.
        *
        * @param    b
        *           The byte to be written.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract void writeByte(byte b)
       throws IOException;

       /**
        * Prints a piece of text without newline at the end. Used for PUT statement in batched mode.
        *
        * @param output
        *        The piece of text to be printed. This should be a formatted and pre-arranged piece of text
        *        obtained in <code>Stream.putWorker()</code>.
        */
       public void writeInline(String output)
       {
           BatchHelper.preventGuiBatchStdout();

           ThinClient tc = ThinClient.getInstance();
           tc.flushTerminal();
           tc.setIsNewLineNeeded(true);
           System.out.print(output);
       }

       /**
        * Write the given string to the output stream.
        *
        * @param    data
        *           The data to be written.
        */
       public abstract void write(String data)
       throws IOException;

       /**
        * Write the given byte array to the output stream.
        *
        * @param    data
        *           The data to be written.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract void write(byte[] data)
       throws IOException;

       /**
        * Write the specified range of bytes from the given byte array to the output stream.
        *
        * @param    data
        *           The data to be written.
        * @param    off
        *           Starting offset in data from which to read bytes to be written. Must be
        *           non-negative and {@code &lt; data.length}.
        * @param    len
        *           Length of data to be written. Must be non-negative and {@code &lt;= (data.length
        *           - offset)}.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public abstract void write(byte[] data, int off, int len)
       throws IOException;

       /**
        * Peeks at the character from the current read position in the stream (reads a character from
        * the current read position in the stream without incrementing stream read position. The next
        * {@code peekCh()} and {@code readCh()} will return the same value).
        * <p>
        * The underlying stream subclass determines the content of the result. Byte oriented streams
        * such as pipes or files will return a byte while streams that generate keystrokes or
        * characters may return a DBCS or Unicode character.
        *
        * @return  The next character read from the stream, -1 on any failure and -2 upon an
        *          {@code EOF}.
        */
       public abstract int peekCh();

       /**
        * Read a character from the current read position in the stream and increment stream pointer.
        * <p>
        * The underlying stream subclass determines the content of the result.
        * Byte oriented streams such as pipes or files will return a byte while
        * streams that generate keystrokes or characters may return a DBCS or
        * Unicode character.
        *
        * @return   The next character read from the stream, -1 on any failure
        *           and -2 upon an <code>EOF</code>.
        */
       public abstract int readCh();

       /**
        * Read all characters from the current read position in the stream to the
        * next line separator (as determined by the <code>File.separator</code>
        * or to the <code>EOF</code>. Any line separator character(s) and the
        * <code>EOF</code> character are not returned.
        *
        * @return   The next line read from the stream.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        * @throws   EOFException
        *           If this input stream reaches the end before reading all the bytes.
        * @throws   InterruptedException
        *           If any thread interrupted the current thread before or while
        *           the current thread was waiting for a notification.
        */
       public abstract String readLn()
       throws EOFException,
              IOException,
              InterruptedException;

       /**
        * Closes the input stream and releases OS resources associated with it.
        */
       public abstract void closeIn();

       /**
        * Closes the output stream and releases OS resources associated with it.
        */
       public abstract void closeOut();

       /**
        * Closes both the input and output streams and releases OS resources
        * associated with it.
        */
       public abstract void close();

       /**
        * State of the input side of the stream.
        *
        * @return   <code>true</code> if the input side of the stream is active.
        */
       public abstract boolean isIn();

       /**
        * State of the output side of the stream.
        *
        * @return   <code>true</code> if the output side of the stream is active.
        */
       public abstract boolean isOut();

       /**
        * Read a single byte from the underlying stream.
        * <p>
        * The default implementation delegates the read to {@link #readCh()}.  TODO: fix this for {@link DirStream},
        * {@link ProcessStream} and other sub-classes.
        *
        * @return   A single byte from the stream.
        */
       public abstract int readByte();

       /**
        * Checks if the stream is alive or not.
        *
        * @return   <code>true</code> if the stream is valid.
        */
       public boolean isValid()
       {
          return isIn() || isOut();
       }

       /**
        * State of the suppress form feed flag.
        *
        * @return   copy of the flag
        */
       public boolean isSuppressFormFeed()
       {
          return suppressFormFeed;
       }

       /**
        * Assigns the internal stream reference to the given reference.
        *
        * @param    stream
        *           The new internal stream reference to use for all operations.
        *
        * @throws   UnsupportedOperationException
        *           If the requested operation is not supported.
        */
       public abstract void assign(Stream stream)
       throws UnsupportedOperationException;

       /**
        * Read a chunk of bytes from the current read position in the stream to the
        * offset calculated by given length or to the <code>EOF</code>, whichever
        * comes first. The <code>EOF</code> character is not returned.
        *
        * @param    len
        *           The number of bytes to read.
        *
        * @return   The chunk read from the stream or <code>null</code> if there
        *           is nothing to read (the <code>EOF</code> has been reached). The
        *           length of the returned array will be the smaller of the
        *           <code>len</code> parameter or the actual bytes left before the
        *           <code>EOF</code>.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public byte[] readBytes(int len)
       throws IOException
       {
          byte[] chunk = new byte[len];

          for (int i = 0; i < len; i++)
          {
             int next = readByte();

             switch (next)
             {
                case -1:
                   // some problem
                   String er = String.format("Unknown failure reading at offset %d", i);
                   throw new IOException(er);
                case -2:
                   if (i == 0)
                   {
                      // the first read reached EOF
                      return null;
                   }
                   else
                   {
                      // just return whatever was read
                      return Arrays.copyOf(chunk, i);
                   }
                default:
                   chunk[i] = (byte) next;
                   break;
             }
          }

          return chunk;
       }

       /**
        * Read a chunk of characters from the current read position in the stream to the
        * offset calculated by given length or to the <code>EOF</code>, whichever
        * comes first. The <code>EOF</code> character is not returned.
        *
        * @param    len
        *           The number of characters to read.
        *
        * @return   The chunk read from the stream or <code>null</code> if there
        *           is nothing to read (the <code>EOF</code> has been reached). The
        *           length of the returned array will be the smaller of the
        *           <code>len</code> parameter or the actual bytes left before the
        *           <code>EOF</code>.
        *
        * @throws   IOException
        *           If an I/O error occurs.
        */
       public byte[] readChunk(int len)
       throws IOException
       {
          byte[] chunk = new byte[len];

          for (int i = 0; i < len; i++)
          {
             int next = readCh();

             switch (next)
             {
                case -1:
                   // some problem
                   String er = String.format("Unknown failure reading at offset %d", i);
                   throw new IOException(er);
                case -2:
                   if (i == 0)
                   {
                      // the first read reached EOF
                      return null;
                   }
                   else
                   {
                      // just return whatever was read
                      return Arrays.copyOf(chunk, i);
                   }
                default:
                   chunk[i] = (byte) next;
                   break;
             }
          }

          return chunk;
       }

       /**
        * Returns an indication of whether the current page has received any
        * output yet. Page headers are not considered part of the output.
        * <p>
        * The current column has to be zero indicating there is no pending PUT
        * output.
        *
        * @return   <code>true</code> if the page is fresh.
        */
       public boolean isFreshPage()
       {
          int next = rawGetNextLineNum();
          return (next == 1 || next == headerSize + 1) && (column == 0);
       }

       /**
        * Gets the number of available lines (free space) on the current output
        * page.
        *
        * @return   The number of available lines.  This may be positive, 0 or
        *           negative.  It can be negative in the case where the headers
        *           and/or footers for a page are larger than the page size (a
        *           perfectly valid, if silly, thing in 4GL).
        */
       public int getAvailableLines()
       {
          int total = rawGetPageSize();
          int next  = rawGetNextLineNum();

          // calculate the amount of used lines
          int taken = next - 1 + (column == 0 ? 0 : 1);

          // if we have not output headers yet, account for that
          taken += (next == 1) ? headerSize : 0;

          // exclude footers since they will be output at the end
          int rem = total - taken - footerSize;
          return rem;
       }

       /**
        * Gets the number of lines on the page excluding footers. Since footers
        * consume part of the lines on a paged stream, this method returns the
        * last effective line that can have non-footer output.
        *
        * @return   The effective page size.
        */
       public int getEffectivePageSize()
       {
          return rawGetPageSize() - footerSize;
       }

       /**
        * Force new output page. This action writes footers, if any, on the
        * current page and completes the page with the form feed. Next, it
        * renders all page headers on the new page.
        * <p>
        * This method should be called only when the caller guarantees at least
        * one character will be output to the new page. Otherwise the last headers
        * will be there without reason.
        */
       public void forcePageBreak()
       {
          forcePageBreak(true);
       }

       /**
        * Force new output page. This action writes footers, if any, on the
        * current page and completes the page with the form feed. Depending on
        * the parameter, it may render all page headers on the new page.
        *
        * @param    headers
        *           <code>true</code> to force the headers on the next page to
        *           be output before returning.
        */
       public void forcePageBreak(boolean headers)
       {
          try
          {
             extendPage();
          }
          catch (IOException ioe)
          {
             ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
          }

          writePageBreak();

          if (headers)
          {
             processHeaders();
          }
       }

       /**
        * Sets contiguous mode for the current page. While <code>true</code> no
        * automatic page break will be generated when output reached the end of
        * the page size.
        * <p>
        * At the end of flushing, the flags will be reset.
        *
        * @param    contiguous
        *           <code>true</code> to enable contiguous mode.
        */
       public void setContiguous(boolean contiguous)
       {
          this.contiguous = contiguous;
       }

       /**
        * Gets the contiguous mode state.
        *
        * @return   <code>true</code> is the stream has contiguous mode enabled.
        */
       public boolean isContiguous()
       {
          return contiguous;
       }

       /**
        * Sets the stream font number. The number denotes an index into the font table
        * of the current environment.
        * The method is part of the runtime support of the FONT option of the OUTPUT TO PRINTER
        * (FWD extension to 4GL) statement. This method is only implemented by the printer stream.
        *
        * @param   num
        *          Font number.
        */
       public void setFont(NumberType num)
       {
          if (!num.isUnknown())
          {
             setFont(num.intValue());
          }
       }

       /**
        * Sets the stream font number. The number denotes an index into the font table
        * of the current environment.
        * The method is part of the runtime support of the FONT option of the OUTPUT TO PRINTER
        * statement (FWD extension to 4GL). This method is only implemented by the printer stream.
        *
        * @param   num
        *          Font number.
        */
       public void setFont(int num)
       {
          UnimplementedFeature.unsupported("FONT option is only supported by printer streams.");
       }

       /**
        * Detects if need to make a pulse on a empty redirected frame stream.
        *
        * @return   <code>true</code> if need to call pulse on stream closing.
        */
       public boolean getPulseOnClose()
       {
          return pulseOnClose;
       }

       /**
        * Sets the need to pulse on close flag.
        *
        * @param   pulseOnClose
        *          New value for pulse on close flag.
        */
       public void setPulseOnClose(boolean pulseOnClose)
       {
          this.pulseOnClose = pulseOnClose;
       }

       /**
        * Detects when a page break is needed.
        *
        * @return   <code>true</code> if a page break should be output.
        */
       boolean needsPageBreak()
       {
          return needsPageBreak(0);
       }

       /**
        * Detects when a page break is needed.
        *
        * @param    add
        *           The number of lines of output that are about to be added. If
        *           this value would cause a page to be overflowed, then a page
        *           break will be detected.  May be 0 in the case where this is
        *           called after output is complete.
        *
        * @return   <code>true</code> if a page break should be output.
        */
       boolean needsPageBreak(int add)
       {
          // TODO: should we return false in contiguous mode and/or if the stream is not paged

          // NOTE: rawGetNextLineNum represents the number of the next line to be written on, not the
          //       number of filled lines on current page. The last written line is the previous one:
          //       rawGetNextLineNum() - 1
          int sz     = rawGetPageSize();
          int target = rawGetNextLineNum() - 1 + add;
          int bottom = sz - footerSize;

          boolean rc = sz > 0 && target > bottom;
          return rc;
       }

       /**
        * Gets the current page size in lines.
        *
        * @return   The number of lines per page.
        */
       int rawGetPageSize()
       {
          return pageSize;
       }

       /**
        * Sets the current page size in lines.
        *
        * @param    sz
        *           The number of lines per page.
        */
       void rawSetPageSize(int sz)
       {
          if (sz == SET_PAGED_DEFAULT)
             sz = PAGED_DEFAULT;

          pageSize = sz;
          pageNum = (sz <= 0) ? 0 : 1;
       }

       /**
        * Gets the current page number.
        *
        * @return   The current page number.
        */
       int rawGetPageNum()
       {
          return pageNum;
       }

       /**
        * Increments the current page number.
        */
       void rawIncrementPageNum()
       {
          pageNum++;
       }

       /**
        * Gets the current page's next line number.
        *
        * @return   The next line number.
        */
       int rawGetNextLineNum()
       {
          return nextLineNum;
       }

       /**
        * Increments the current page's next line number.
        */
       void rawIncrementNextLineNum()
       {
          nextLineNum++;
       }

       /**
        * Resets the current page's next line number to 1.
        */
       void rawResetNextLineNum()
       {
          nextLineNum = 1;
       }

       /**
        * Check if this is  a file resource.  Implementations will override this as needed.
        *
        * @return   <code>false</code> as default.
        */
       protected boolean isFileResource()
       {
          return false;
       }

       /**
        * Read a block of data from the stream with the block size equal to the size of the binary data array in
        * the given <code>memptr</code> variable. The data will be written into the memptr starting at the first
        * byte in the array.  The read data will be the smaller of the block size or the file size.  Only an error
        * or end of file condition will cause this read to terminate before the block size has been read.
        *
        * @param    var
        *           The variable to which the result is assigned.
        *
        * @return   The number of the bytes read, -1 on any failure and -2 upon an <code>EOF</code>.
        */
       protected int readBlockRemote(memptr var)
       {
          throw new UnsupportedOperationException();
       }

       /**
        * This method should be called after each write operation. If stream is unbuffered, then
        * call flushData method.
        *
        * @throws   IOException
        *           if an I/O error occurs.
        */
       protected final void afterWrite()
       throws IOException
       {
          if (isUnbuffered())
          {
             flushData();
          }
       }

       /**
        * Flushes this output stream and forces any buffered output bytes to be written out.
        * The general contract of flush is that calling it is an indication that,
        * if any bytes previously written have been buffered by the implementation of the output
        * stream, such bytes should immediately be written to their intended destination.
        *
        * @throws   IOException
        *           if an I/O error occurs.
        */
       protected void flushData() throws IOException
       {
          // no-op
       }

       /**
        * Perform common close <b>preprocessing</b> for all streams (should be
        * called at the <b>beginning</b> of each sub-classes' {@link #close}
        * method.  This allows processing to occur while the stream is still
        * open.
        *
        * @param    in
        *           <code>true</code> if this was triggered during a call to
        *           {@link #closeIn} or {@link #close}.  Input resources are
        *           to be cleaned up.
        * @param    out
        *           <code>true</code> if this was triggered during a call to
        *           {@link #closeOut} or {@link #close}.  Output resources are
        *           to be cleaned up.
        */
       protected void cleanup(boolean in, boolean out)
       throws IOException
       {
          if (in)
          {
             doneIn = true;
          }

          if (out && !doneOut)
          {
             // this call triggers some final output to the stream. so it has to
             // come before turning on doneOut
             processCloseNotifications();

             doneOut = true;

             int pageSz = rawGetPageSize();
             int pageNo = rawGetPageNum();

             try
             {
                // paged output?
                if (pageSz > 0)
                {
                   int next = rawGetNextLineNum();

                   // non-empty page?
                   // if there is output coming from a put statement on line one,
                   // then we still need to output the page-break and footers
                   // if there was output with no trailing NL we still need page
                   // break and footer
                   if (next > 1                   ||
                       (next == 1 && !trailingNL) ||
                       lastPageWithOutput == pageNo)
                   {
                      if (footers != null)
                      {
                         // fill in a partial page with blanks before processing
                         // any footer or form feed
                         extendPage();
                         buffer = null;
                      }

                      // paged output that has a non-empty page always gets
                      // a form feed output at the end of the file
                      writePageBreak();
                   }
                }
             }
             finally
             {
                // flush collected data if any
                endOutput();

                // finally inform listeners that stream has been reached end-of-life
                processUnbindNotifications();
             }
          }
       }

       /**
        * Remove the new line character(s) from the given buffer and return the
        * rest of the buffered text as a string.
        *
        * @param    sb
        *           Buffered text, possibly ended by a new line.
        *
        * @return   Any text which exists less the new line character(s).
        */
       protected String readLineCleanup(StringBuilder sb)
       throws EOFException
       {
          int len = sb.length();

          if (len == 0)
          {
             throw new EOFException();
          }

          int idx = indexOfNewLine(sb);

          if (idx < 0)
          {
             idx = len;
          }
          else if (idx > 0)
          {
             char other = sb.charAt(idx - 1);

             if (other == '\r')
             {
                // some implementations (even on Linux) may render an extra char
                // even though the platform newline is just a '\n', remove this
                idx--;
             }
          }

          // idx == 0 case needs no modification

          // return the data less the newline itself
          return sb.substring(0, idx);
       }

       /**
        * Helper method to search the position of the new line string in a given
        * text buffer. The method tries to locate system default line separator,
        * if there is no match, Linux style line separator and finally the Windows
        * style line separator. It uses in case the input files are came from OS
        * other than one the P2J runtime is working.
        * <p>
        * WARNING: this looks only in the last two characters in the buffer - if the newline is
        * expected to be in the middle of the string, an alternative is needed.
        *
        * @param    sb
        *           Buffered text to find the new line separator string.
        *
        * @return   The position of the new line string or -1 if it is not found.
        */
       protected int indexOfNewLine(StringBuilder sb)
       {
          int l = sb.length();
          if (l == 0)
          {
             return -1;
          }

          String s = (l == 1 ? sb.toString() : sb.substring(l - 2, l));

          int result = -1;
          for (String nl : NEW_LINES)
          {
             // the lookup must be done using the exact order from NEW_LINES, so a line with '\r\n' is matched
             // with '\r\n' and not '\n'.
             result = s.indexOf(nl);
             if (result >= 0)
             {
                break;
             }
          }

          return (result == -1) ? -1 : (l > 1) ? (result + (l - 2)) : result;
       }

       /**
        * Check if these two bytes form a new line.
        *
        * @param    bn_1
        *           First byte.
        * @param    bn
        *           Second byte.
        *
        * @return   <code>true</code> if the bytes correspond for a new line.
        */
       protected boolean isNewLine(int bn_1, int bn)
       {
          // check the last two read bytes and compare them with the new line
          for (String nl : NEW_LINES)
          {
             if (nl.length() == 1 && nl.charAt(0) == bn)
             {
                return true;
             }

             if (nl.length() == 2 && nl.charAt(0) == bn_1 && nl.charAt(1) == bn)
             {
                return true;
             }
          }

          return false;
       }

       /**
        * Indicate whether or not a lone hyphen on input should be ignored;
        * normal IMPORT statement processing considers this a blank field.
        * Subclasses which require the hyphen to be read as a normal character
        * should override this method.
        *
        * @return  <code>true</code>.
        */
       protected boolean skipLoneHyphenInput()
       {
          return true;
       }

       /**
        * Output a list of fields to the stream.
        *
        * @param    data
        *           The list of fields to be output.
        * @param    mode
        *           <code>MODE_PUT_NORMAL</code>, <code>MODE_PUT_CONTROL</code>,
        *           <code>MODE_PUT_UNFORMATTED</code> or <code>MODE_EXPORT</code>.
        * @param    delim
        *           The delimiter to use in EXPORT mode.
        *
        * @throws   ErrorConditionException
        *           On any error during I/O.
        */
       public void putWorker(FieldEntry[] data, int mode, char delim)
       throws ErrorConditionException
       {
          boolean put = (mode != MODE_EXPORT);

          try
          {
             switch (mode)
             {
                case MODE_PUT_CONTROL:
                   setControl(true);
                   break;
                case MODE_PUT_UNFORMATTED:
                   setUnformatted(true);
                   break;
                case MODE_EXPORT:
                   setDelimiter(delim);
                   break;
             }
             isPut = put;

             if (put)
             {
                // TODO: the nested error mode flag is only used on the server side, we really should only call
                //       this if we are on the server; at this time this causes the TM to be loaded on the client
                //       side which can cause unexpected init of various compents

                // enable nested error mode
                TransactionManager.setNestedMode(true);
             }

             if (flushBeforePut || !isFileResource())
             {
                // only non-file resources force the flush on each PUT or EXPORT statement.
                flush();
             }

             // we save the put variable so we can use it if we need
             // to restart the output
             savedPut = put;

             // protection from buffer closing by external writer
             synchronized (outActivityLock)
             {
                putWorkerActive = true;
             }

             // init our buffer and reset column for EXPORT mode
             startOutput(put ? column : 0);

             // output each field
             for (FieldEntry aData : data)
             {
                // we check for page-break before each field is output
                checkForPageBreak();
                if (aData != null)
                {
                   aData.output(this);
                }
             }

             if (!put)
             {
                // now it is safe to close buffer
                synchronized (outActivityLock)
                {
                   putWorkerActive = false;
                }
                // flush our buffer and explicitly add NL
                endOutput(true);
             }
             else if (!control)
             {
                // flush output immediately
                try
                {
                   String output = currentOut.toString();
                    if (this instanceof NullStream &&
                        OutputManager.isBatchInBackground())
                   {
                       writeInline(output);
                   }
                   else
                   {
                      write(output);
                   }
                   currentOut.setLength(0);

                   BufferSizeManager.getInstance().notifyOutputOperation(
                                                                 true,
                                                                 this,
                                                                 output.length());
                }
                catch (IOException ioe)
                {
                   ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
                }
                finally
                {
                   // now it is safe to close buffer
                   synchronized (outActivityLock)
                   {
                      putWorkerActive = false;
                   }
                }
             }
             else
             {
                // now it is safe to close buffer
                synchronized (outActivityLock)
                {
                   putWorkerActive = false;
                }
             }
          }
          catch (StopConditionException err)
          {
             // pipe has been broken
             throw err;
          }
          catch (ConditionException err)
          {
             if (put)
             {
                // allow caller to see the error (silent mode will never be
                // active so this will always throw an error)
                ErrorManager.recordOrThrowError(564,
                                                "Unable to evaluate expression for PUT statement");
             }
             else
             {
                // allow caller to see the original error directly in export mode
                throw err;
             }
          }

          finally
          {
             if (put)
             {
                // TODO: only call this on the server

                // disable nested error mode
                TransactionManager.setNestedMode(false);
             }

             switch (mode)
             {
                case MODE_PUT_CONTROL:
                   setControl(false);
                   break;
                case MODE_PUT_UNFORMATTED:
                   setUnformatted(false);
                   break;
                case MODE_EXPORT:
                   setDelimiter(null);
                   break;
             }
             isPut = false;
             flushBeforePut = false;
          }
       }

       /**
        * Check if current stream is bound to terminal.
        *
        * @return   <code>true</code> if stream is terminal.
        */
       public boolean isTerm()
       {
          return false;
       }

       /**
        * Number of lines pending for output but not buffered in local stream
        * buffer.
        *
        * @return  Number of pending lines.
        */
       public int getPendingRowCount()
       {
          return pendingRowCount;
       }

       /**
        * Notify stream about pending rows.
        *
        * @param    rows
        *           Number of pending rows.
        */
       public void setPendingRowCount(int rows)
       {
          pendingRowCount = rows;
       }

       /**
        * Read a block of data from the stream with the block size equal to the size of the binary data array in
        * the given <code>memptr</code> variable. The data will be written into the memptr starting at the first
        * byte in the array.  The read data will be the smaller of the block size or the file size.  Only an error
        * or end of file condition will cause this read to terminate before the block size has been read.
        * <p>
        * This API is used by the FWD client-side.
        *
        * @param    var
        *           The variable to which the result is assigned.
        *
        * @return   The number of the bytes read, -1 on any failure and -2 upon an <code>EOF</code>.
        */
       int readBlockWorker(memptr var)
       {
          // TODO: memptr may be on server-side, improve this

          long len = var.lengthOf();
          byte[] bytes = readBytesWorker(len);

          if (bytes == null)
          {
             return -1; // unexpected error
          }

          if (bytes.length == 0)
          {
             return -2; // EOF
          }
          else
          {
             var.setBytes(new raw(bytes), 1);
          }

          return bytes.length;
       }

       /**
        * Read a chunk of bytes from the specified stream, of at most the given length.
        *
        * @param   len
        *          The maximum length of bytes to read.
        *
        * @return  {@code null} in case of errors or the read bytes otherwise.
        */
       private byte[] readBytesWorker(long len)
       {
          byte[] res = new byte[(int) len];
          int readLen = -1;

          for (int i = 0; i < len; i++)
          {
             int ch = readByte();
             if (ch == -1)
             {
                return null;
             }

             if (ch == -2)
             {
                readLen = i;
                break;
             }

             res[i] = (byte) ch;
          }

          if (readLen != -1 && readLen != len)
          {
             byte[] res2 = new byte[readLen];
             System.arraycopy(res, 0, res2, 0, readLen);
             res = res2;
          }

          return res;
       }

       /**
        * Extend the output of a partial page (with blank lines) to the end of
        * valid output so that any footer and a form feed can be output to
        * terminate the page at the proper minimum length.  This page extend
        * feature only occurs if footers are being used.
        */
       private void extendPage()
       throws IOException
       {
          if (footers != null)
          {
             int len = (rawGetPageSize() - footerSize);
             int from = rawGetNextLineNum();

             for (int i = from;  i <= len; i++)
             {
                write(NEWLINE);
                rawIncrementNextLineNum();
             }
          }
       }

       /**
        * Obtain the next line of input and translate any errors or exceptional
        * conditions into their Progress equivalents.
        *
        * @param    inString
        *           <code>true</code> if the special EOF processing (when the '.'
        *           appears by itself on its own line) should NOT be honored.
        *           This only occurs during formatted reading when a string is
        *           being processed.
        *
        * @return   The next line of input or <code>null</code> in the case of
        *           and error condition raised during silent error mode.
        *
        * @throws   EndConditionException
        *           Upon reading at EOF (not up to EOF but the read that occurs
        *           where EOF is the next character to be read).
        * @throws   ErrorConditionException
        *           If an IOException occurs.
        * @throws   StopConditionException
        *           If the read was interrupted.
        */
       private String reload(boolean inString)
       throws EndConditionException,
              ErrorConditionException,
              StopConditionException
       {
          String next = null;

          try
          {
             next = readNextLine(inString, false);
          }
          catch (EOFException eof)
          {
             throw new EndConditionException("EOF");
          }
          catch (InterruptedIOException | InterruptedException iioe)
          {
             ErrorManager.handleStopException(new StopConditionException(iioe));
          }
          catch (IOException ioe)
          {
             // TODO: actual Progress compatible text and number are unknown
             ErrorManager.recordOrThrowError(-1, "IOException during read.");
             return null;
          }

          return next;
       }

       /**
        * Read the next line from the stream using Progress rules. This must
        * properly handle the <code>EOF</code> event (as reported by the stream),
        * the <code>EOF</code> character (0x1A or CTRL-Z), the fake Progress
        * <code>EOF</code> character (a "." by itself on its own line) and the
        * line continuation characters of tilde and backslash.
        * <p>
        * If the <code>null</code> character is detected in the read data, all
        * data from that character through the next <code>newline</code>
        * (inclusive) will be discarded.  This is effectively, another form of
        * line continuation as the next line will be seamlessly concatenated to
        * this result. The only exception to this is when there is no following
        * <code>newline</code> on the NEXT line in which case ALL data read is
        * discarded and the empty string is returned (WARNING: the EOF event is
        * not raised until the next call to this method in this case).
        *
        * @param    inString
        *           <code>true</code> if the special EOF processing (when the '.'
        *           appears by itself on its own line) should NOT be honored.
        *           This only occurs during formatted reading when a string is
        *           being processed.
        *
        * @param    isImportUnformatted
        *           <code>true</code> if the import has the unformatted option.
        *
        * @return   The next line from the stream.
        */
       private String readNextLine(boolean inString, boolean isImportUnformatted)
       throws EOFException,
              IOException,
              InterruptedException
       {
          // reset flag
          nullQuirk = false;

          StringBuilder sb = new StringBuilder();

          String next = readLineDropNull();

          if (next == null)
          {
             throw new EOFException("");
          }

          // process all line continuations
          while (next.endsWith("~") || next.endsWith("\\"))
          {
             sb.append(next, 0, next.length() - 1);
             next = readLineDropNull();
          }

          // add the only segment or the last segment
          sb.append(next);

          String res = sb.toString();

          if ((res.equals(".") && !inString && !isImportUnformatted) ||
              (res.length() == 1 && res.charAt(0) == EOF))
          {
             throw new EOFException("");
          }

          return res;
       }

       /**
        * Read the next line from the stream dropping all data at and after any
        * <code>null</code> character including the next <code>newline</code>.
        * This is effectively, another form of line continuation as the next line
        * will be seamlessly concatenated to this result.
        * <p>
        * The only exception to this is when there is no following
        * <code>newline</code> on the NEXT line in which case ALL data read is
        * discarded and the empty string is returned (WARNING: the EOF event is
        * not raised until the next call to this method in this case). Note that
        * in this case an internal flag is set which can be checked in the
        * caller.
        *
        * @return   The next line from the stream.
        */
       private String readLineDropNull()
       throws EOFException,
              IOException,
              InterruptedException
       {
          // this may be null or throw EOFException in the case of EOF (the
          // behavior depends on the backing stream implementation)
          String next = readLn();

          if (next != null)
          {
             int idx = next.indexOf('\0');

             if (idx != -1)
             {
                // embedded null case so truncate the current line
                StringBuilder sb = new StringBuilder(next.substring(0, idx));

                try
                {
                   // continue data with the next line
                   next = readLineDropNull();
                }
                catch (EOFException eof)
                {
                   // eat this because we must be able to silently return in the
                   // case where there is no line delimiter on the NEXT line after
                   // an embedded null case occurs
                   next = null;
                }

                if (next == null)
                {
                   nullQuirk = true;
                }
                else
                {
                   sb.append(next);
                }

                next = sb.toString();
             }
          }

          return next;
       }

       /**
        * Add a line end and if in paged output mode and the current page is
        * full, add a page break (the form feed character 0x0C) to the current
        * write buffer and maintain the next line number and current page number.
        * <p>
        * If in control mode, no page breaks will be emitted and the page/line
        * counters will not be modified.
        * <p>
        * If <code>conditional</code> parameter is set to <code>true</code> then
        * new line is appended only if buffer does contain new line at the end.
        *
        * @param    conditional
        *           If this parameter is set to <code>true</code> then appending
        *           of new line is conditional.
        * @param    noPageBreak
        *           If this parameter is set to <code>true</code> then there is no
        *           checking for page break. Instead, the line-counter is
        *           incremented.
        */
       private void processLineEnd(boolean conditional, boolean noPageBreak)
       throws ErrorConditionException
       {
          if (!conditional || !trailingNL)
          {
             writeNewLine();

             if (noPageBreak)
             {
                rawIncrementNextLineNum();
             }
             else
             {
                processPageBreak();
             }
          }
       }

       /**
        * Add the platform-specific new line to the current write buffer.
        */
       private void writeNewLine()
       throws ErrorConditionException
       {
          append(NEWLINE);

          if (!control)
          {
             column = 0;
          }
       }

       /**
        * Set the NEWLINE state.
        *
        * @param    trailingNL
        *           Track trailing NEWLINE state.
        */
       private void setTrailingNL(boolean trailingNL)
       {
          this.trailingNL = trailingNL;
          this.trailingWithoutPutNL = trailingNL & (!isPut);
       }

       /**
        * If in paged output mode and the current page is full, add a page break
        * (the form feed character 0x0C) to the current write buffer and maintain
        * the next line number and current page number.
        * <p>
        * If in control mode, no page breaks will be emitted and the page/line
        * counters will not be modified (this is a NOP).
        */
       private void processPageBreak()
       {
          if (!control)
          {
             // determine if we are in paged mode AND if we should be starting the next page
             if (needsPageBreak())
             {
                // move to a new page
                writePageBreak();
             }
             else
             {
                // just increment our counter
                rawIncrementNextLineNum();
             }
          }
       }

       /**
        * Add any needed footer(s) and then a page break (the form feed character
        * 0x0C) to the current stream (buffered or not buffered), then increment
        * the current page number and reset the next line number to 1.
        */
       private void writePageBreak()
       throws ErrorConditionException
       {
          if (footersEmitted)
          {
             // if the footers were output to the stream, this means the
             // page-break was also written - so we do nothing more.
             return;
          }

          // output footer
          if (footers != null)
          {
             processFooters();
          }

          try
          {
             // output form feed
             if (!suppressFormFeed)
             {
                try
                {
                   if (currentOut == null)
                   {
                      try
                      {
                         // unbuffered mode
                         write("\f");
                      }
                      catch (IOException ioe)
                      {
                         ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
                         return;
                      }
                   }
                   else
                   {
                      // buffered output
                      append("\f");
                   }
                }
                finally
                {
                   // maintain paging state
                   rawIncrementPageNum();
                   rawResetNextLineNum();
                   setTrailingNL(true);
                }
             }
          }
          finally
          {
             // send notifications
             processPageBreakNotifications();

             suppressFormFeed = false;
          }
       }

       /**
        * Notify all registered listeners that a page break has been written for
        * this stream.
        * Notified listeners are automatically removed from the registry.
        */
       private void processPageBreakNotifications()
       {
          if (pageListeners != null)
          {
             // iterate on the copy so that the notification handler can re-register if needed
             Map<PageBreakListener, Object> copy = new HashMap<>(pageListeners);
             Iterator<PageBreakListener> list = copy.keySet().iterator();

             while (list.hasNext())
             {
                PageBreakListener pl = list.next();
                pl.notifyPageBreak(this);
                pageListeners.remove(pl);
             }
          }
       }

       /**
        * Notify all registered listeners that this stream is being closed.
        */
       private void processCloseNotifications()
       {
          if (closeListeners == null)
             return;

          for (CloseListener cl : closeListeners.keySet())
          {
             cl.notifyClose(this);
          }
       }

       /**
        * Notify all registered listeners that this stream is dead.
        */
       private void processUnbindNotifications()
       {
          if (closeListeners == null)
             return;

          for (CloseListener cl : closeListeners.keySet())
          {
             cl.notifyUnbind(this);
          }

          closeListeners.clear();
          closeListeners = null;
       }

       /**
        * Notify all registered listeners that this stream is advancing page.
        */
       private void processAdvanceNotifications()
       {
          if (advanceListeners != null)
          {
             Iterator<AdvanceListener> list = advanceListeners.iterator();

             while (list.hasNext())
             {
                AdvanceListener al = list.next();
                al.notifyAdvance(this);
                list.remove();
             }
          }
       }

       /**
        * Iterate through the given set of {@link RepeatedOutput} instances and
        * for each element, invoke the {@link RepeatedOutput#render} method.
        *
        * @param    out
        *           The list of instances to use for rendering.
        * @param    footers
        *           When set to <code>true</code>, it means we output the page
        *           footers; else, we output the headers.
        */
       private void processRendering(Set<RepeatedOutput> out, boolean footers)
       {
          // don't let the processing of headers OR of footers trigger recursive
          // emission of footers
          footersEmitted = true;

          if (out == null)
             return;

          // casting works only if streams are not using rpc
          StreamDaemon streamHelper = (StreamDaemon) OSResourceManager.getStreamHelper();

          // make sure the output is switched to this stream temporarily
          int sid = streamHelper.idFromStream(this);
          int oid = -2;

          ThinClient tc = ThinClient.getInstance();
          if (!tc.isRedirected(this))
          {
             oid = tc.switchOutput(sid, false);
          }

          Iterator<RepeatedOutput> list = out.iterator();
          RedirectedTerminal rt = ThinClient.redirectedTerminalForStream(this);

          while (list.hasNext())
          {
             RepeatedOutput ro = list.next();
             ro.render(this);

             // flush after every page element
             rt.flush();

             try
             {
                flushControlled(true);
             }
             catch (IOException ioe)
             {
             }

             // if we output the headers, then the column field must be set to zero - this is because
             // the put statement must output on a new line, not near the page-break character
             if (!footers)
             {
                column = 0;
             }
          }

          if (oid != -2)
          {
             tc.switchOutput(oid, false);
          }

          footersEmitted = false;
       }

       /**
        * Append the given string to the output buffer and maintain the column
        * counter. If in <code>control</code> mode, the output buffer is
        * bypassed (the output is written directly to the stream) and no updates
        * are made to the line/page counters.
        *
        * @param    data
        *           The string to append.
        */
       private void append(String data)
       throws ErrorConditionException,
              StopConditionException
       {
          setTrailingNL(data.endsWith(NEWLINE));

          if (!control)
          {
             column += data.length();
             currentOut.append(data);
          }
          else
          {
             try
             {
                write(data);
             }

             catch (InterruptedIOException iioe)
             {
                ErrorManager.handleStopException(new StopConditionException(iioe));
             }

             catch (IOException ioe)
             {
                ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
             }
          }
       }

       /**
        * Append the given <code>char</code> to the output buffer and maintain
        * the column counter.  If in <code>control</code> mode, the output buffer is
        * bypassed (the output is written directly to the stream) and no updates
        * are made to the line/page counters.
        *
        * @param    data
        *           The <code>char</code> to append.
        */
       private void append(char data)
       throws ErrorConditionException,
              StopConditionException
       {
          setTrailingNL(NEWLINE_LEN == 1 && NEWLINE.charAt(0) == data);

          if (!control)
          {
             column += 1;
             currentOut.append(data);
          }
          else
          {
             try
             {
                writeCh(data);
             }
             catch (InterruptedIOException iioe)
             {
                ErrorManager.handleStopException(new StopConditionException(iioe));
             }
             catch (IOException ioe)
             {
                ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
             }
          }
       }

       /**
        * Generates formatted data in conformity with PUT / PUT UNFORMATTED / PUT
        * CONTROL specifications.
        * <p>
        * Formatting rules are specified in the table below, but please read
        * comments to the table first:
        * <ul>
        *    <li>
        *      By the words "specific format" we mean one of the following
        *      (in preference order):
        *     <ol>
        *       <li>
    	 *          explicit format string that was specified into PUT or
        *          PUT UNFORMATTED statement;
        *       </li>
        *       <li>
    	 *          explicit format string that was specified into variable
        *          definition statement;
        *       </li>
        *       <li> default type-specific format. </li>
        *     </ol>
        *    </li>
        *    <li>
        *       By the words "left-aligned up to some length" we mean that
        *       trailing spaces were added to the formatted value in order to
        *       match given length.
        *    </li>
        *    <li>
        *       By the words "right-aligned up to some length" we mean that
        *       leading spaces were added to the formatted value in order to match
        *       given length.
        *    </li>
        *    <li>
        *       By the words "string format" we mean string representation of the
        *       instance data in a form that is compatible with the MESSAGE
        *       language statement.
        *    </li>
        * </ul>
        * <p>
        * <table style="text-align: left; width: 100%;" border="1" summary="">
        * <tr>
        *    <td></td>
        *    <td style="font-weight: bold;">
        *       put (with or without format string) / put unformatted with format
        *       string specification
        *    </td>
        *    <td style="font-weight: bold;">
        *       put unformatted without format string specification / put control
        *    </td>
        * </tr>
        * <tr>
        *    <td> integer, not unknown </td>
        *    <td> specific format, right-aligned up to format length </td>
        *    <td> string format </td>
        * </tr>
        * <tr>
        *    <td> decimal, not unknown </td>
        *    <td> specific format, right-aligned up to format length </td>
        *    <td> string format </td>
        * </tr>
        * <tr>
        *    <td> character, not unknown </td>
        *    <td> specific format</td>
        *    <td> string format </td>
        * </tr>
        * <tr>
        *    <td> date, not unknown </td>
        *    <td> specific format</td>
        *    <td> string format </td>
        * </tr>
        * <tr>
        *    <td> logical, not unknown </td>
        *    <td>
        *       specific format, left-aligned up to maximum between positive
        *       value length and negative value length
        *    </td>
        *    <td> string format </td>
        * </tr>
        * <tr>
        *    <td> recid, not unknown </td>
        *    <td> specific format, right-aligned up to format length </td>
        *    <td> string format </td>
        * </tr>
        * <tr>
        *    <td> unknown integer </td>
        *    <td> ?, right-aligned up to specific format length </td>
        *    <td> ? </td>
        * </tr>
        * <tr>
        *    <td> unknown decimal </td>
        *    <td> ?, right-aligned up to specific format length </td>
        *    <td> ? </td>
        * </tr>
        * <tr>
        *    <td> unknown character </td>
        *    <td> ?, left-aligned up to specific format length </td>
        *    <td> ? </td>
        * </tr>
        * <tr>
        *    <td> unknown date </td>
        *    <td>
        *       string consisting of spaces which number equals specific format
        *       length
        *    </td>
        *    <td> ? </td>
        * </tr>
        * <tr>
        *    <td> unknown logical </td>
        *    <td>
        *       ?, left-aligned up to maximum between positive value length and
        *       negative value length in the specific format
        *    </td>
        *    <td> ? </td>
        * </tr>
        * <tr>
        *    <td> unknown recid </td>
        *    <td> ?, right-aligned up to specific format length </td>
        *    <td> ? </td>
        * </tr>
        * <tr>
        *    <td> unknown literal </td>
        *    <td>
        *       if explicit format string is specified, then it is ?, left-aligned
        *       up to explicit format length, otherwise it is ? followed by three
        *       spaces
        *    </td>
        *    <td> ? </td>
        * </tr>
        * </table>
        *
        * @param   val
        *          The data to be output.
        * @param   fmt
        *          The format string to use. If <code>null</code>, the default
        *          type-specific format will be used.
        *
        * @return  Formatted data.
        */
       private String generateFormattedText(BaseDataType val, String fmt)
       {
          String text = null;

          if ((val instanceof logical) && (fmt == null && !unformatted))
          {
             fmt = val.defaultFormatString();
          }

          if ((!unformatted || fmt != null) && !control)
          {
             String defFormat = null;
             if (val instanceof character)
             {
                defFormat = generateStringFormat(null, val.toStringMessage().length());
             }
             else
             {
                defFormat = val.defaultFormatString();
             }

             boolean isNumberType =  val instanceof NumberType;

             boolean isDefaultFormat = fmt != null && fmt.equals(defFormat);

             if (isPut && isDefaultFormat && !(isNumberType))
             {
                //do not use default format, only explicit format is allowed in unformatted mode
                text = val.toStringMessage();
             }
             else
             {
                text = val.toString(fmt);
             }
          }
          else if (val instanceof raw && !val.isUnknown())
          {
             text = new String(((raw) val).getByteArray());
          }
          else
          {
             text = val.toStringMessage();
          }

          if ((!unformatted || fmt != null) && !control)
          {
             if (val instanceof NumberType)
             {
                int len = (fmt == null) ? val.defaultFormatString().length()
                                        : NumberType.getValueTextLength(text, fmt);

                text = StringHelper.rightAlignText(text, len);
             }
             else if (((val instanceof logical) || (val instanceof character)) && val.isUnknown())
             {
                int len = val.formatLength(fmt);
                text = StringHelper.leftAlignText(text, len);
             }
             else if ((val instanceof date) && val.isUnknown())
             {
                int len = (fmt == null) ? val.defaultFormatString().length() : fmt.length();
                text = StringHelper.repeatChar(' ', len);
             }
             else if (val instanceof unknown)
             {
                int len = (fmt == null) ? 4 : val.formatLength(fmt);
                text = StringHelper.leftAlignText("?", len);
             }
          }

          return text;
       }

       /**
        * This method is a custom algorithm for page-break checking - the skip
        * option with a put stream ... statement will emit the page-break only if
        * <code>LINE-COUNTER &ge; PAGE-SIZE + 1</code>
        */
       private void checkForPageBreak()
       {
          int pageSz = rawGetPageSize();
          int lineCounter = rawGetNextLineNum();
          int bottom = pageSz - footerSize;

          // a page break will be emitted only if we are beyond line PAGE-SIZE + 1
          if (pageSz > 0 && lineCounter >= bottom + 1)
          {
             writePageBreak();

             // we need to check if page headers are needed
             processHeaders();

             // re-start the output, if the footers or headers end it
             if (currentOut == null)
             {
                startOutput(savedPut ? column : 0);
             }
          }
       }
    }