Stream.java

/*
** Module   : Stream.java
** Abstract : represents a Progress compatible stream for reading/writing
**
** Copyright (c) 2005-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- ----------------------------------Description-----------------------------------
** 001 GES 20050712   @21742 Created initial version supporting methods
**                           for the standardized open/close, and both
**                           formatted and unformatted reading/writing
**                           of streams. Also has facilities for seek,
**                           line and page processing.
** 002 GES 20050801   @21941 New assign() interface for support of shared streams.
** 003 GES 20050824   @22214 Implemented error processing including
**                           silent error mode. Implemented the
**                           Finalizable interface to provide for
**                           automatic cleanup.
** 004 GES 20050907   @22593 Added support for the memptr type (see
**                           readBlock() and writeBlock()).
** 005 ECF 20051104   @23236 Added no-op iterate() method. Required to
**                           satisfy expanded Finalizable interface.
** 006 GES 20060115   @23921 Changed readLine to support BDT not just
**                           character.
** 007 GES 20060207   @24328 Translate InterruptedException into a
**                           StopConditionException.
** 008 GES 20060208   @24393 Added echo support and cleaned up docs.
** 009 GES 20060217   @24669 Added writeByte().
** 010 GES 20060225   @24743 Added footer and header support.
** 011 GES 20060226   @24752 Added methods to remote the paging state.
** 012 GES 20060227   @24772 Final footer/header support.
** 013 GES 20060302   @24857 Minor changes to enable removal from the
**                           finalizable lists.
** 014 ECF 20060330   @25261 Throw EOFException from readNextLine() when
**                           no next line is available.
** 015 GES 20060412   @25489 Added putField() signatures.
** 016 GES 20060517   @26194 Match new Finalizable interface requirement.
** 017 GES 20060523   @26455 Fix for skipping multiple sequential
**                           delimiters in readFieldWorker() instead of
**                           returning an empty string.
** 018 GES 20060817   @28662 Fix to ensure that all paged streams get a
**                           form feed at the end.  Previously this was
**                           only done in the cleanup processing for
**                           streams with footers.
** 019 GES 20060905   @29226 Fixes for embedded null byte processing in
**                           field reading.
** 020 GES 20060925   @29905 Added high level equivalents to the PUT and
**                           EXPORT language statements instead of 
**                           requiring the caller to implement the start
**                           and end buffering and the individual field
**                           output.
** 021 GES 20061004   @30135 Added support for bracketing PUT in nested
**                           error mode enable/disable processing.  This
**                           duplicates a quirk in PUT processing where
**                           a nested error will be honored after the
**                           original error has been processed (so it is
**                           as if there are 2 errors rather than 1 as
**                           would normally be the case).
** 022 GES 20061102   @30906 Added hasPageHeadersFooters() query method.
** 023 GES 20061113   @31101 Modified the redirected terminal output
**                           processing to always group buffered output
**                           together (it now keeps all this output on
**                           the same page, even though that may cause the
**                           resulting page size to be smaller or larger 
**                           than specified for the stream).
** 024 GES 20061115   @31169 Added a page break notification framework.
** 025 NVS 20070516   @33545 Page headers and footers may do registrations
**                           multiple times. Fixed the headers and footers
**                           cumulative size calculation to allow it.
** 026 NVS 20070601   @33918 Page break listeners are removed from the
**                           registry after they process notification.
** 027 NVS 20070611   @34008 Added pulseOutput() method and protected
**                           processHeaders() method from recursion.
** 028 NVS 20070716   @34542 Fixed putUnformatted() method which should
**                           output NL. Added registerCLoseListener()
**                           method and stream close notification for all
**                           registered listeners.
** 029 NVS 20070716   @34561 #028 was insufficient to solve the new line
**                           issue with putUnformatted(), so I've added
**                           variations of methods that take a flag that
**                           controls the auto-append of new line.
** 030 NVS 20070724   @34678 Fixed close notification processing (doneOut)
**                           and added buffer flushing after headers and
**                           footers.
** 031 NVS 20070725   @34683 Added isIn() and isOut() abstract methods.
** 032 NVS 20070730   @34737 Redirected mode flushes terminal's buffers
**                           after every page element.
** 033 NVS 20070731   @34755 Added registerAdvanceListener() method. Used
**                           to notify all registered listeners about
**                           page advance being performed.
** 034 NVS 20070802   @34767 Fixed advancePage() method which should check
**                           the page line number after sending page 
**                           advance notification.
** 035 NVS 20070802   @34776 Added registerFlushListener() and flush() 
**                           methods. They provide a way to notify all
**                           registered listeners about flushing being 
**                           performed on the stream.
** 036 NVS 20070828   @34923 Removed some debugging leftovers.
** 037 SIY 20070829   @34924 Unknown date should be represented as spaces
**                           in output (Ref: @34927). 
** 038 NVS 20070906   @34972 Since now page elements rendering can be done
**                           while switching streams for a frame,
**                           processRendering() makes sure a proper
**                           redirection is established for the duration
**                           of this rendering.
** 039 SIY 20070914   @35075 Fixed handling of some cases of line feeds.
**                           Ref: #35073
** 040 SIY 20070924   @35197 Deeper rework of line feeds handling.
**                           Ref: #35198
** 041 SIY 20070925   @35202 Fixed regression caused by #39: EXPORT 
**                           immediately after PUT inserts redundant 
**                           delimiter before first field.
**                           Ref: #35199#1, #35201.  
** 042 SIY 20070928   @35326 Fix for PUT/DISPLAY handling regressions.
**                           Ref: #35314
** 043 EVL 20071005   @35374 Removing finalize() method. We can not control
**                           when it is called.
** 044 EVL 20071008   @35396 Fixed redundant NL's.
** 045 EVL 20071010   @35410 Added the method to check the validity of
**                           the stream.
** 046 NVS 20071018   @35483 Fixed a bug with delayed flushing of the
**                           current output buffer putting Form Feeds far
**                           away from the true page break point.
** 047 SIY 20071018   @35490 Fixes in NL handling in control mode.
**                           Ref: #35486
** 048 SIY 20071023   @35498 Fixes regression caused by change in NL 
**                           handling (#047).
** 049 NVS 20071025   @35520 An attempt to solve the problem of split
**                           output due to page overflow. Now
**                           flushControlled() redraws page headers
**                           internally and merges the recursively created
**                           output in the proper order.
** 050 SIY 20071028   @35609 Fixed NPE.
**                           Ref: #35579.
** 051 SIY 20071030   @35629 Fixed other regression caused by (#047).
**                           Page break should be handled only when
**                           NL is inserted. Also, changed default trailing
**                           NL flag to true.
** 052 SIY 20071106   @35714 Minor change in javadoc.
** 053 SIY 20071116   @35823 Do not include trailing empty line when
**                           checking for page break.
**                           Ref: #35800.
** 054 CA  20071127   @36033 Added field newLineRequired used to output a 
**                           NL just before the page break. Changed access
**                           modifier for method processHeaders from 
**                           private to public. Ref: #36031
** 055 NVS 20071212   @36321 Introduced the frozen list of page headers.
**                           Registrations and deregistrations are done
**                           against a live list, which gets frozen by
**                           means of producing a snapshot of the list.
**                           It's the frozen list that supplies headers
**                           info for the stream when processing page
**                           breaks. Freezing happens with the very first
**                           output operation by calling a newly added
**                           method freezeHeaders(). Once processed, the
**                           frozen list gets destroyed and should be
**                           recreated for the new page similarly.
** 056 GES 20071218   @36429 Removed unnecessary member.
** 057 CA  20071219   @36543 The skip option inside a put statement is
**                           executed on the same page and will check for
**                           page-break before and after it is executed. 
**                           The cleanUp method must take into account if 
**                           a put statement output some data on the first
**                           line of a page, without advancing the cursor
**                           to the next line (this is needed to output  
**                           correct the footers and the page-break).
**                           Fixed page-headers/footers generation when  
**                           put statements are used. Ref: 36542
** 058 NVS 20071229   @36619 Introduced contiguous output mode and means
**                           to control it. Made flushControlled() a void
**                           method. Fixed page headers with advancePage
**                           method (PAGE statement).
**                           Added getAvailableLines(), isFreshPage() and
**                           forcePageBreak() methods.
** 059 GES 20080103   @36660 Back out H055. That change was meant to solve
**                           an issue where there was a registered page
**                           element and pending output in the redirected
**                           terminal buffer and at the time of flushing
**                           that buffer, the page element had been
**                           removed so the output from that element was
**                           never seen.  In the 4GL the output was seen.
**                           The essential problem is that the page
**                           elements need to be forced to output at the
**                           moment the first output is made for a page.
**                           We were deferring that output too long. The
**                           H055 attempted to solve this problem by
**                           remembering the page element list at the
**                           moment of first output.  But this change
**                           causes a great deal of breakage in other
**                           use cases AND it doesn't solve all possible
**                           issues. The correct solution has the
**                           redirected terminal call processHeaders()
**                           on the associated stream when it is adding
**                           output for the first time to a fresh buffer.
** 060 CA  20080103   @36655 If the buffer size is p = PAGE-SIZE -
**                           LINE-COUNTER + 1, then flushControlled() must
**                           write the buffer content on the same page, as  
**                           there are p lines available (the line on row
**                           LINE-COUNTER is included too in the output).
**                           Ref: #36540
** 061 CA  20080107   @36699 Back out the H059 change in extendPage() - 
**                           is OK to start the counter from LINE-COUNTER.
** 062 GES 20080104   @36702 advancePage now supports extendPage when
**                           page footers are in use.
** 063 CA  20080108   @36723 endOutput() should increment the LINE-COUNTER
**                           if a NL is appended. Ref: #36706
**                           flushControlled() needs to compare the 
**                           remaining lines against the actual buffer 
**                           size. Ref: #36705
** 064 CA  20080115   @36816 Added method processFooters() which will 
**                           process the footers in a LIFO order.
** 065 GES 20080108   @36723 Changes to make flushing more correct. In
**                           particular, many cases of inappropriate
**                           recursion in page header/footer processing
**                           were eliminated. It should never be possible
**                           for header processing to trigger footer
**                           processing or vice versa. Exposed some state
**                           and new controls regarding paging and
**                           flushing.
** 066 CA  20080118   @36850 Fixed page-break output when flushing a
**                           buffer with size 2 and ending with a NL and
**                           there are not enough rows in the current page 
**                           for the buffer content.
** 067 SIY 20080125   @36913 Fixed PUT SKIP(n) with n != 0 for streams 
**                           without explicitly set page size.
**                           Ref: #36912
** 068 CA  20080303   @37233 Added 'unnamed' field, which is true if this
**                           is an unnamed stream. Ref: #37225
** 069 EVL 20080312   @37433 Changed the approach of calculation pageNum.
**                           This fixes the difference between 4gl and p2j
**                           when the real report output is empty.
** 070 SIY 20080317   @37455 Added support for terminal stream.
**                           Ref: #37453
** 071 GES 20080403   @37826 Added interface for readField(BDT[]).
** 072 GES 20080412   @38021 Added partial support for the convert option.
**                           Fixed readFieldWorker() to handle embedded
**                           newlines in strings and to use a more
**                           appropriate conversion from export format
**                           (the previous approach used a worker that
**                           did more than just stripping the enclosing
**                           quotes and replacing the doubled double
**                           quotes with single double quotes).
** 073 GES 20080418   @38046 Added raw support to readField(). Fixed
**                           edge cases for readFieldWorker() to allow
**                           input buffer extension when a quote char is
**                           just before the embedded newline (either a
**                           start of string quote char or the 2nd in a
**                           doubled up pair of escaped quotes).
** 074 SIY 20080418   @38033 Do not throw an error condition if some
**                           amount of data was read successfully.
**                           Ref: #38032
** 075 GES 20080425   @38123 Refactored to eliminate round trips to the
**                           client by passing more data in a single
**                           invocation. This improves performance.
** 076 SIY 20080515   @38285 Extend page when page break is forced by
**                           view() or down().
** 077 SIY 20080515   @38286 Improved detection of presence output on the
**                           page. Since page break resets trailingNL flag
**                           to initial value (true), then any output 
**                           without NL at the page start will set it to 
**                           false. Any output with NL will increase line
**                           counter, so presence of output will still be 
**                           correctly detected.
** 078 GES 20080519   @38302 Disable the special '.' EOF marker when
**                           parsing a string from the readFieldWorker.
** 079 SIY 20080721   @39151 Updated to accommodate changes in logical 
**                           toString() behavior.
** 080 GES 20080730   @39236 Fix for setPosition() to handle a new
**                           position located past the end of the file
**                           in the same way as the 4GL. The previous
**                           version incorrectly handled this.
** 081 NVS 20080812   @39373 Added support for redirected editors output
**                           implemented in new writeControlledSpecial()
**                           method. Refactored shared code into worker
**                           methods.
** 082 SVL 20090208   @41268 readField(FieldReference) method added.
** 083 SVL 20090209   @41274 generateFormattedText() behavior fixed.
** 084 SIY 20090220   @41340 Added isTerm() method which by default return
**                           false (means 'not a terminal').
** 085 GES 20090422   @41915 Converted to standard string formatting.
** 086 GES 20090424   @41973 Import change.
** 087 GES 20090501   @42090 Notify listeners about stream end-of-life.
** 088 SVL 20090225   @41368 Into generateFormattedText() changed hardcoded
**                           length of default format for the NumberType to a
**                           dynamic one, in order to support recid type.
** 089 SVL 20090612   @42673 Added "unbuffered" flag. Stop condition (rather
**                           than error condition) is raised on output errors.
**                           Notify BufferSizeManager about PUT operations.
** 090 GES 20090624   @42885 Protect stream reads, writes and other state
**                           changes/inspections from closed streams.
** 091 ECF 20090709   @43127 Fixed memory leak. Made pageBreakListeners and
**                           closeListeners sets into WeakHashMaps to ensure
**                           unreferenced listeners are garbage collected in
**                           the absence of an explicit deregistration
**                           mechanism. Note that this changes the iteration
**                           order of these collections from FIFO (since these
**                           were previously implemented as LinkedHashSets) to
**                           undefined. However, this appears to be safe.
** 092 SVL 20090815   @43622 Skip multiple sequential delimiters in
**                           readFieldWorker() only if delimiter is space.
**                           Parse empty strings as integer, decimal or date
**                           without any errors.
** 093 SIY 20090907   @43834 Explicitly closed stream should not complain about
**                           writes into closed stream.
** 094 SIY 20090925   @44034 Added pendingRowCount memebr and accessor. Now 
**                           getNextLineNum() takes pending rows into account.
** 095 SVL 20110403          Added write(byte[]) function.
** 096 GES 20110812          Added readChunk() method.
** 097 ECF 20121219          Added skipLoneHyphenInput method and modified
**                           readFieldWorker to make skipping a lone hyphen
**                           encountered on input dependent upon the return
**                           value from this new method. This is meant to
**                           allow subclasses to override this default
**                           behavior.
** 098 EVL 20130116          Added indexOfNewLine() method to make finding
**                           the new line separator strings the system neutral.
**                           The method readLineCleanup has been changed to
**                           use this tool.
** 099 CS  20130319          Added conversion support for Landscape and Portrait IO options.
** 100 OM  20130406          Added support for 64 bit operations.
** 101 GES 20130523          Matched signature changes in BinaryData and used a more efficient
**                           approach than a byte-by-byte writing loop. 
** 102 OM  20130607          Fixed datetime/-tz deserialization (fromLiteral).
** 103 CA  20131013          Added no-op deleted() method, required by the changes in Finalizable
**                           interface.
** 104 EVK 20140207          In generateFormattedText() added special case for RAW type.
** 105 EVL 20140418          Fixes for different format variations.
** 106 EVK 20140523          Added unbuffered stream operation.
** 107 CA  20140730          Protected stream close/cleanup code from raised conditions, to allow
**                           close/cleanup fully complete even if conditions were raised.
** 108 GES 20141030          Added setConvertSource/Target() methods.
** 109 ECF 20150715          Replace StringBuffer with StringBuilder.
** 110 EVL 20160224          Javadoc fixes to make compatible with Oracle Java 8 for Solaris 10.
** 111 EVK 20161123          Disable formatting for put unformatted output.
** 112 OM  20161209          Fixed needsPageBreak() computation. Upgrade code to Java8.
** 113 OM  20170612          Granted protected access to [binary] field. Fixed readChar() in
**                           non-binary case.
** 114 CA  20170827          Added stub for readField(ImportFieldExpansion).
**     HC  20170904          Modified conversion of IMPORT statement to expand extent fields
**                           dynamically. This should prevent the Java source file size to get out
**                           of control for large extents.
** 115 HC  20171024          Runtime support for SYSTEM-DIALOG PRINT-SETUP and OUTPUT TO PRINTER.
** 116 ECF 20171026          Added write(byte[], int, int) method. Refactored readField to make
**                           core data assignment accessible statically.
** 117 EVL 20180620          Adding pulse on close flag support.
** 118 EVL 20180831          Fix for NPE when concurrent thread attempts to close the buffer after
**                           short message when main worker is still in progress.
** 119 ECF 20180818          Avoid use of expensive String.format.
** 120 SBI 20181106          Changed generateFormattedText in order to format a number according
**                           to its format.
** 121 EVL 20190423          Adding runtime support for stream input-output code page conversions.
**                           Now by default we do conversion from -cpinternal to -cpstream for
**                           output and from -cpstream to -cpinternal for input operations.
**                           Adding MAP/NO-MAP options flag runtime supprort.
**     GES 20190509          Reworked code to cache cp conversion decision (for performance) and
**                           to remove code duplication.
** 122 CA  20190511          Added support for IMPORT ... UNFORMATTED {record|field|OO property}.
** 123 CA  20190906          Improved performance of indexOfNewLine - it searches only the last 
**                           two characters, and not the entire string buffer.
** 124 CA  20200211          Allow fractional seconds (convert to millis based on the legacy 
**                           version).
** 125 RFB 20200419          Added support for KEEP-MESSAGES to be able to suppress some messages
**                           from the default display.
**     CA  20200427          Added support for blob and clob fields.
** 126 RFB 20200710          indexOfNewLine was not taking into account when the only character in the buffer
**                           is the newline itself. It was returning -1, which indicates not found.
**     GES 20200820          Added some comments.
**     CA  20210304          Fixed date, datetime and datetimetz literal parsing. 
**     EVL 20210309          Adding raw data type as valid destination for readLine(BDT) method for case when
**                           we read text file in binary mode into raw buffer variable.
**     EVL 20210310          Adding more importers for longchar, blob, clob, memptr, recid and rowid data
**                           types to be able to read into via readLine(BDT) method.  Fixed String line import
**                           into binary based data types.
**     EVL 20210311          Small changes to improve compatibility for binary based data types import with
**                           readLine(BDT) method calling to read binary data file.
**     CA  20210317          Improved performance of the readBlock(memptr) API.
**     OM  20210328          OE quirk: if last line is not EOL terminated, the readLn() method returns the
**                           previous read line (actually, if that EOL is missing the file is binary by
**                           definition).
**     OM  20210401          Reverting previous commit. It seems it is not the correct way to handle it.
**     EVL 20210402          Fix for raw variable set up bug while readLine().  We need to make raw final
**                           variable size matching the line length to avoid \0 chars keeeping in final raw.
**                           Also experimental feature to add new line chars when reading raw via String.
**     EVL 20210405          Improved readLine() processing for raw data type.  All line are now reading.
**                           The new line separators are embedding into final data buffer.  Taking into
**                           account different values of initial raw variable size and incoming data length.
**                           Also OS behind and binary mode is considering to make the final data buffer.
**     EVL 20210406          Fixed the bug when partial reading into raw variable drops any further reading
**                           from the current position of the same stream into another variable.
**     CA  20210422          Reworked 'IMPORT UNFORMATTED raw/memptr/blob' - although it relies on the 
**                           readLine() API in the conversion, the runtime will read the stream byte-by-byte, 
**                           and not 'per-line'.
**     CA  20210531          Fixed a NPE in 'IMPORT UNFORMATTED lob' - the bytes need to be written, not the
**                           'result'.
**     OM  20210602          Support for multibyte character sets. Use CPSTREAM for default char encoding.
**     CA  20210709          Added APIs to read pristine bytes.  Characters are returned and decoded only by
**                           the readCh() and readLine() APIs.
**     CA  20210919          When reading from a file, the "?" literal must be treated as a string, and the ?
**                           literal must be treated as an unknown value.
**     CA  20210920          Fixed a regression in reading a line (largest separator needs to be checked first).
**                           Hard-coded the possible new lines instead of computing them.
**     CA  20211001          Do not flush file resources on PUT/EXPORT statements.
**     AL2 20211217          Byte 26 should stop reading only in non-binary Windows.
**     EVL 20220325          Base refactoring for Stream based classes getting single byte and array of bytes.
**     IAS 20220408          Fixed support for PUT CONTROL <raw>.
**     CA  20220515          memptr and library calls are supported on server-side, too.  Added TODO's for 
**                           client-side APIs which will need to call back to server-side, when memptr is used
**                           as argument.
**     HC  20220824          Stream now initiates flush when there is buffered data in the related
**                           RedirectedTerminal. This prevents data directly written to the Stream with
**                           PUT statement to be written to the target sooner than the buffered data in
**                           RedirectedTerminal.
**     EVL 20221207          Adding method to set up LOB directory for Stream object.
**     EVL 20221212          The runtime implementation is only in FileStream class.
**     EVL 20221222          Adding runtime support for LOB fields export.  Merges changes from OM for
**                           readField() method.
** 128 PMP 20240403          Added import unformatted flag to readNextLine method.
**     GES 20240412          Minor clarification in javaodc.
** 129 PMP 20240509          Modified putWorker method to check for null field entries.
** 130 SB  20240617          Use isWindows instead of recalling EnvinronmentOps.isUnderWindows.
** 131 GBB 20240912          Replace ThinClient access to streams with StreamDaemon from OSResourceManager.
** 132 FER 20241118          Added the option to print to console if PUT is performed in batched mode. putWorker()
**                           method. This is done through the new method writeInline().
**                           Added import OutputManager.
** 133 FER 20250304          Changed the if from putWorker to make sure that Window batch processes only print to
**                           stdout when not redirected to files.
** 134 AI  20250228          Improved performance of readLine by buffering the read of bytes.
** 135 ES  20250424          Handle StopConditionException through ErrorManager.
** 136 AI  20250528          Fixup extra read of byte when max length of bytes was read.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util;

import java.io.*;
import java.nio.charset.*;
import java.util.*;
import com.goldencode.p2j.ui.LogicalTerminal;
import com.goldencode.p2j.ui.client.OutputManager;
import com.goldencode.p2j.ui.client.chui.driver.batch.BatchHelper;
import com.goldencode.p2j.util.osresource.*;
import com.goldencode.util.*;
import com.goldencode.p2j.ui.chui.RedirectedTerminal;
import com.goldencode.p2j.ui.chui.ThinClient;
import com.goldencode.p2j.persist.FieldReference;
import com.goldencode.p2j.schema.*;

/**
 * A stream class supporting methods for the Progress-compatible reading and
 * writing as well as facilities for line and page processing.  This class
 * defines a common API for read/write operations while implementing the
 * Progress semantics that are common to all streams.  In particular, the  
 * formatting, options processing and all knowledge of the wrapper data
 * types is limited to this class.
 * <p>
 * Subclasses implement the specific read/write/seek workers that are
 * specific to the type of stream they represent.  These workers generally
 * pass exceptions through to the caller and this class handles those
 * exceptions that can be handled in a standard manner.  The design is to
 * ensure that the worker methods can be standardized and can be implemented
 * with little effort.  The following methods must be implemented:
 * <p>
 * <pre>
 * {@link #assign}
 * {@link #available}
 * {@link #getPos}
 * {@link #setPos}
 * {@link #getLen}
 * {@link #setLen}
 * {@link #writeByte}
 * {@link #writeCh}
 * {@link #write}
 * {@link #readCh}
 * {@link #readLn}
 * {@link #closeIn}
 * {@link #closeOut}
 * {@link #close}
 * </pre>
 * <p>
 * Please note that many of these abstract methods may simply throw the
 * <code>UnsupportedOperationException</code> if the underlying stream does
 * not support the operation in question.
 * <p>
 * Language statements that are supported:
 * <p>
 * <pre>
 * Language Statement      Java Equivalent
 * ---------------------   ----------------------
 * define stream           {@link #Stream} default constructor
 * export                  {@link #export} (uses {@link #writeField} or {@link #writeBlock})
 * import                  {@link #readField} or {@link #readLine} or {@link #readBlock} 
 * input close             {@link #closeIn}
 * input from              Stream subclass constructor
 * input-output close      {@link #close}
 * output close            {@link #closeOut}
 * output to               Stream subclass constructor
 * page                    {@link #advancePage}
 * put                     {@link #put} or {@link #putControl} or {@link #putUnformatted} (uses {@link #putField})
 * readkey                 {@link #readChar}
 * seek                    {@link #setPosition}
 * </pre>
 * <p>
 * Built-in functions that are supported:
 * <p>
 * <pre>
 * Function                Java Equivalent
 * ---------------------   ----------------------
 * line-counter            {@link #getNextLineNum}
 * page-number             {@link #getPageNum}
 * page-size               {@link #getPageSize}
 * seek                    {@link #getPosition}
 * </pre>
 * Language statements that have backing/helper support but which are fully 
 * implemented elsewhere:
 * <p>
 * <pre>
 * Language Statement      Uses
 * ---------------------   ----------------------
 * prompt-for              {@link #readFieldWorker}
 * set                     {@link #readFieldWorker}
 * update                  {@link #readFieldWorker}
 * </pre>
 * <p>
 * Direct use of the {@link #putField}, {@link #writeField} and 
 * {@link #writeBlock} output methods generate output to a buffer which must
 * first be initialized by {@link #startOutput}.  Once initialized, any number
 * of calls to such output methods may be done and all output will be gathered
 * into the same output buffer.  To write this buffer to the stream, call
 * {@link #endOutput}. Calls to the output methods must never be done outside
 * of a bracketed pair of matching <code>startOutput</code> and
 * <code>endOutput</code> calls otherwise the results will be undefined.
 * <p>
 * Within a given bracketed pair of matching <code>startOutput</code> and
 * <code>endOutput</code> calls, please do not mix the output of 
 * <code>putField</code> and <code>writeField</code>.  The results are
 * undefined.
 * <p>
 * Use of {@link #put}, {@link #putControl}, {@link #putUnformatted} and
 * {@link #export} DO NOT require the use of {@link #startOutput} or
 * {@link #endOutput} since they all handle buffering internally.
 * <p>
 * This class honors silent error mode (see {@link ErrorManager}) if this
 * has been enabled.  Otherwise, the end or error conditions are raised
 * (the appropriate exceptions are thrown). <b>At this time, the error text
 * and error numbers are not fully implemented (many of the Progress
 * compatible values are unknown). In addition, the state of the stream
 * may be different from the Progress behavior in cases where an error is
 * raised.</b>
 *
 * @author    GES
 */
public abstract class Stream
implements Finalizable
{
   /**
    * The value returned by {@link #readFieldWorker(BaseDataType)} when the exact <code>?</code> character
    * was read. 
    */
   public static final Object UNKNOWN_VALUE = new Object();
   
   /** Enum for representing the stream print modes*/
   public static enum PrintMode { LANDSCAPE, PORTRAIT, NONE };
   
   /** The default delimiter to use for field level record reading. */
   public static final char DELIM = ' ';   
   
   /** The line separator character(s) for the current platform. */
   public static final String NEWLINE = System.getProperty("line.separator");
   
   /** The length if line separator. */
   public static final int NEWLINE_LEN = NEWLINE.length();
   
   /**
    * The new lines, ordered by their length.  The order is important as a string with '\r\n' must match
    * with '\r\n' and not '\n'.
    */
   public static final String[] NEW_LINES = new String[] { "\r\n", "\n" };
   
   /** Special value which triggers setting of default number of lines per page in paged mode. */
   public static final int SET_PAGED_DEFAULT = -1;
   
   /** The default number of lines per page when in paged mode (56). */
   public static final int PAGED_DEFAULT = 56;
   
   /** The <code>EOF</code> character. */
   public static final char EOF = (char) 0x1A;
   
   /** The Carriage Return character {@code CR = 0x0D}. */
   public static final int CR = 0x0D;
   
   /** The Line Feed character {@code LF = 0x0A}. */
   public static final int LF = 0x0A;
   
   /** The Carriage Return character and Line Feed character */
   public static final String CRLF = "\r\n";
   
   /** {@link #putWorker} mode for a standard PUT statement. */
   public static final int MODE_PUT_NORMAL = 0;
   
   /** {@link #putWorker} mode for a PUT CONTROL statement. */
   public static final int MODE_PUT_CONTROL = 1;
   
   /** {@link #putWorker} mode for a PUT UNFORMATTED statement. */
   public static final int MODE_PUT_UNFORMATTED = 2;
   
   /** {@link #putWorker} mode for an EXPORT statement. */
   public static final int MODE_EXPORT = 3;
   
   /** Message that indicates that pipe to subprocess has been broken. */
   public static final String PIPE_BROKEN_MESSAGE = "** Pipe to subprocess has been broken";
   
   /** Number of milliseconds to pause between checks for input. */
   protected static final int increment = 200;
   
   /** Flag denoting if input resources have already been cleaned up. */
   protected boolean doneIn = false;
   
   /** Flag denoting if output resources have already been cleaned up. */
   protected boolean doneOut = false;
   
   /** External option value for standard character translations occur or not. */
   protected boolean convert = true;
   
   /** Internal value. Determines if standard character translations occur or not. */
   protected boolean _convert = false;
   
   /** Determines if codepage conversion decision is cached. */
   protected boolean convertCached = false;
   
   /** The Legacy source codepage for character conversion. */
   protected String sourceCp = null;
   
   /** The current source charset. This is never {@code null}. */
   protected Charset sourceCharset;
   
   /** Target codepage for character conversion.  Is {@code null} if no conversion was specified. */
   protected String targetCp = null;
   
   /** The current source charset. Is {@code null} if no conversion was specified. */
   protected Charset targetCharset;
   
   /** Default value of the -cpstream option. */
   protected static final String streamCp = I18nOps._getCPStream();
   
   /** Default value of the -cpinternal option. */
   protected static final String internalCp = I18nOps._getCPInternal();
   
   /** Determines if control character translations occur or not. */
   protected boolean binary = false;
   
   /** Flag to keep NO-MAP stream option. */
   protected boolean noMap = false;

   /** Variable to keep MAP stream option. */
   protected String mapEntryProtermcap = null;

   /** Current page number for the output stream. */
   private int pageNum = 0;
   
   /** Lines of output per page or 0 for non-paged output.  */
   private int pageSize = 0;
   
   /** Line number of the next line to be output on the current page. */
   private int nextLineNum = 1;
   
   /** Current input line being processed when reading fields. */
   private String currentIn = null;
   
   /** Current index position into the line being read. */
   private int indexIn = 0;
   
   /** Current output line being processed when writing fields. */
   private StringBuilder currentOut = null;
   
   /** Flag indicating active put worker operation to avoid buffer closing. */
   private boolean putWorkerActive = false;
   
   /** Lock to sync the putWorkerActive access. */
   private final Object outActivityLock = new Object();
   
   /** Current column position on the line being written. */
   private int column = 0;
   
   /** Flag denoting if the output buffer is currently being used. */
   private boolean active = false;   
   
   /** The print display mode of the stream */
   private PrintMode displayMode = PrintMode.NONE;
   
   /** Flag denoting if the output is unformatted. */
   private boolean unformatted = false;   
   
   /** Flag denoting if the UI functions that read input should echo it. */
   private boolean echo = true;   
   
   /** Flag denoting if output modifies line/page counters. */
   private boolean control = false;   
   
   /** Flag to detect if a dropped data occurred in line reading. */
   private boolean nullQuirk = false;
   
   /** Character used for delimiting import and export fields. */
   private char delim = DELIM;   
   
   /** List of headers that are to be output (valid in paged mode). */
   private Set<RepeatedOutput> headers = null;   
   
   /** Size of current headers to be output in lines. */
   private int headerSize = 0;   
   
   /** List of footers are to be output (valid in paged mode). */
   private Set<RepeatedOutput> footers = null;   
   
   /** Size of current footers to be output in lines. */
   private int footerSize = 0;   
   
   /** Intermediate buffer of redirected terminal output. */
   private ArrayList<String> buffer = null;
   
   /** Nesting level for {@link #flushControlled} method invocations. */
   private int flushLevel = 0;   
   
   /** Objects to receive a notification when a page break is written. */
   private Map<PageBreakListener, Object> pageListeners = null;
   
   /** Objects to receive a notification when a closeOut is issued. */
   private Map<CloseListener, Object> closeListeners = null;
   
   /** Objects to receive a notification when a page advance is issued. */
   private Set<AdvanceListener> advanceListeners = null;
   
   /** Objects to receive a notification when a flush is issued. */
   private Set<FlushListener> flushListeners = null;
   
   /** Flag indicating headers being output */
   private boolean inHeaders = false;
   
   /** Track trailing NEWLINE state */
   private boolean trailingNL = true;
   
   /** Track trailing NEWLINE state */
   private boolean trailingWithoutPutNL = true;
   
   /** Field indicating what page had output last */
   private int lastPageWithOutput = -1;
   
   /** 
    * The flag saves the put parameter in the putWorker method; used if we 
    * need to restart the output.
    */
   private boolean savedPut = false;
   
   /** Flag indicating a PUT statement is being used. */
   private boolean isPut = false;
   
   /** 
    * When the flag is true, the page-footers were emitted.
    */
   private boolean footersEmitted = false;
   
   /** 
    * While this flag is set, the current page will be written to ignoring
    * a page overflow condition.
    */
   private boolean contiguous = false;
   
   /** If <code>true</code>, this is an unnamed stream. */   
   private boolean unnamed = false;
   
   /** 
    * When the flag is true due to editor(s), the next form feed will be skipped.
    */
   private boolean suppressFormFeed = false;
   
   /** Determines whether the stream is UNBUFFERED */
   private boolean unbuffered = false;
   
   /** Name of the stream variable used in the legacy 4GL source code. */ 
   private String name = null;
   
   /** Number of rows pending in RemoteTerminal. */
   private int pendingRowCount = 0;
   
   /** Flag to mark pulse on close action. */
   private boolean pulseOnClose = false;
   
   /** Flag denoting if error, warning and message statements are suppressed. */
   private boolean keepMessages = false;
   
   /** Check if we are under Windows OS. */
   protected final boolean isWindows = EnvironmentOps.isUnderWindowsFamily();
   
   /** readByte batch size */
   private final int BATCH_SIZE = 1024;

   /**
    * Flag to direct this stream to call {@linkplain #flush()} in {@link #putWorker(FieldEntry[], int, char)}.
    */
   private boolean flushBeforePut;
   
   /**
    * Default constructor which does nothing.
    */
   public Stream()
   {
      this(null);
   }
   
   /**
    * Construct an instance associated with a specific legacy 4GL variable name.
    *
    * @param    name
    *           The legacy variable name.
    */
   public Stream(String name)
   {
      this.name = name;
      setConvertSource(streamCp); // always starting with CPSTREAM as default for stream encoding/decoding
   }
   
   /**
    * Assign a value derived from a string into a data wrapper variable.
    * 
    * @param   var
    *          Variable into which datum is assigned.
    * @param   value
    *          String value of the datum to be assigned.
    * @param   iso
    *          The {@code value} string is in ISO mode.
    */
   public static void assignDatum(BaseDataType var, String value, boolean iso)
   {
      if (var != null && value != null)
      {
         boolean emptyRes = value.isEmpty();
         
         if (var instanceof blob)
         {
            if (!emptyRes)
            {
               ((blob) var).assign(value.getBytes());
            }
         }
         else if (var instanceof clob)
         {
            ((clob) var).assign(emptyRes ? "" : new character(value));
         }
         else if (var instanceof integer)
         {
            // integer must come before int64. Otherwise the operator instanceof will match 
            // the integer instances, too
            ((integer) var).assign(emptyRes ? new integer(0) : new integer(value));
         }
         else if (var instanceof int64)
         {
            ((int64) var).assign(emptyRes ? new int64(0) : new int64(value));
         }
         else if (var instanceof decimal)
         {
            ((decimal) var).assign(emptyRes ? new decimal(0) : new decimal(value));
         }
         else if (var instanceof datetimetz)
         {
            // datetimetz must come before datetime and date Otherwise the operator instanceof 
            // will match the datetimetz instances, too
            if (emptyRes)
            {
               var.setUnknown();
            }
            else
            {
               ((datetimetz) var).assign(
                     iso ? datetimetz.parseLiteral(value) : datetimetz.fromLiteral(value));
            }
         }
         else if (var instanceof datetime)
         {
            // datetime must come before date Otherwise the operator instanceof will match the 
            // datetime instances, too
            if (emptyRes)
            {
               var.setUnknown();
            }
            else
            {
               ((datetime) var).assign(
                     iso ? datetime.parseLiteral(value) : datetime.fromLiteral(value));
            }
         }
         else if (var instanceof date)
         {
            if (emptyRes)
            {
               var.setUnknown();
            }
            else
            {
               ((date) var).assign(iso ? date.parseLiteral(value) : new date(value));
            }
         }
         else if (var instanceof logical)
         {
            ((logical) var).assign(new logical(value));
         }
         else if (var instanceof character)
         {
            ((character) var).assign(new character(value));
         }
         else if (var instanceof raw)
         {
            ((raw) var).assign(new raw(value));
         }
      }
   }
   
   /**
    * Accesses the line number of the next line to be output on the current page.
    *
    * @return   The next line number to be output.
    */
   public integer getNextLineNum()
   {
      if (protectWrites())
      {
         return new integer(0);
      }
      
      return new integer(rawGetNextLineNum() + getPendingRowCount());
   }   
   
   /**
    * Accesses the line number of the next line to be output on the current page.
    *
    * @return   The next line number to be output.
    */
   public int rawNextLineNum()
   {
      return rawGetNextLineNum();
   }   
   
   /**
    * Accesses the current page number in an output stream.  The return value
    * is undefined for an input stream.
    *
    * @return   The current page number.
    */
   public integer getPageNum()
   {
      if (protectWrites())
         return new integer(0);
      
      return new integer(rawGetPageNum());
   }   
   
   /**
    * Accesses the number of lines per page in an output stream, where 0
    * indicates that this stream is not paged.  The return value is undefined
    * for an input stream.
    *
    * @return   The number of lines per page of output or 0 if the output
    *           is not paged.
    */
   public integer getPageSize()
   {
      if (protectWrites())
         return new integer(0);
      
      return new integer(rawGetPageSize());
   }   
   
   /**
    * Accesses the number of lines per page in an output stream, where 0
    * indicates that this stream is not paged.  The return value is undefined
    * for an input stream.
    *
    * @return   The number of lines per page of output or 0 if the output
    *           is not paged.
    */
   public int rawPageSize()
   {
      return rawGetPageSize();
   }   
   
   /**
    * Sets the number of lines per page in an output stream, where 0
    * indicates that this stream is not paged.
    *
    * @param    size
    *           The number of lines per page of output or 0 if the output
    *           is not paged.  If the input value is <code>unknown</code>
    *           no change will be made.
    */
   public void setPageSize(NumberType size)
   {
      if (!size.isUnknown())
      {
         setPageSize(size.intValue());
      }
   }   
   
   /**
    * Sets the number of lines per page in an output stream, where 0
    * indicates that this stream is not paged.
    *
    * @param    size
    *           The number of lines per page of output or 0 if the output
    *           is not paged.
    */
   public void setPageSize(double size)
   {
      int sz = (int) size;
      
      rawSetPageSize((sz <= 0) ? 0 : sz);
   }   
   
   /**
    * Sets the number of lines per page in an output stream to the
    * {@link #PAGED_DEFAULT} or to terminal height for {@link TerminalStream}.
    */
   public void setPageSize()
   {
      rawSetPageSize(SET_PAGED_DEFAULT);
   }   
   
   /**
    * Sets the stream into binary mode which disables character translations
    * during {@link #readChar}.  It is important to note that field and line
    * reading (<code>import</code>, <code>set</code>, <code>update</code>
    * <code>prompt-for</code> and <code>import unformatted</code>) are
    * unaffected by this setting (this is based on Progress testcases).
    */
   public void setBinary()
   {
      binary = true;
   }
   
   /**
    * Sets the stream NO-MAP option causing to completely ignore PROTERMCAP file while data
    * transformation.
    */
   public void setNoMap()
   {
      noMap = true;
   }
   
   /**
    * Sets the stream MAP option causing to use PROTERMCAP file entry while data transformation.
    */
   public void setMap(String protermcapEntry)
   {
      // TODO: implement the real work
      mapEntryProtermcap = protermcapEntry;
   }
   
   /**
    * Sets the stream's character conversion mode.
    *
    * @param    convert
    *           The new character conversion mode.  <code>true</code> to
    *           enable character conversion, <code>false</code> to disable
    *           conversion.
    */
   public void setConvert(boolean convert)
   {
      this.convert = convert;
   }
   
   /**
    * Gets the stream's character conversion mode.
    *
    * @return   The convert mode. <code>true</code> means that character
    *           conversion is enabled.
    */
   public boolean getConvert()
   {
      return convert;
   }
   
   /**
    * Sets the stream's source codepage and activates character conversion mode.
    *
    * @param   cp
    *          The source codepage for character conversion mode.
    */
   public void setConvertSource(String cp)
   {
      // skip for null 
      if (cp != null)
      {
         // protect from setting to empty or just whitespaced strings
         cp = cp.trim();
         if (!cp.isEmpty())
         {
            sourceCp = cp;
         }
      }
      else
      {
         sourceCp = I18nOps.ABL_LATIN1; // default to Latin-1 / ISO-8859-1 
      }
      
      String javaCS = I18nOps.getJavaCharset(sourceCp);
      if (javaCS == null)
      {
         javaCS = I18nOps.JAVA_LATIN1; // TODO: maybe throw exception instead
         sourceCp = I18nOps.ABL_LATIN1; // to reflect the actual value
      }
      
      this.sourceCharset = Charset.forName(javaCS);
   }
   
   /**
    * Sets the stream's source codepage and activates character conversion mode.
    *
    * @param    cp
    *           The source codepage for character conversion mode.
    */
   public void setConvertSource(character cp)
   {
      if (cp == null || cp.isUnknown())
      {
         UnimplementedFeature.todo("CONVERT SOURCE: confirm unknown handling is correct.");
         return;
      }
      
      setConvertSource(cp.toStringMessage());
   }
   
   /**
    * Sets the stream's target codepage and activates character conversion mode.
    *
    * @param    cp
    *           The target codepage for character conversion mode.
    */
   public void setConvertTarget(String cp)
   {
      // skip for null 
      if (cp != null && !cp.trim().isEmpty())
      {
         // protect from setting to empty or just whitespaced strings
         cp = cp.trim();
         if (!cp.isEmpty())
         {
            targetCp = cp;
         }
      }
      else
      {
         targetCp = I18nOps.ABL_LATIN1; // default to Latin-1 / ISO-8859-1 
      }
      
      String javaCS = I18nOps.getJavaCharset(targetCp);
      if (javaCS == null)
      {
         javaCS = I18nOps.JAVA_LATIN1; // TODO: maybe throw exception instead
         targetCp = I18nOps.ABL_LATIN1; // to reflect the actual value
      }
      
      this.targetCharset = Charset.forName(javaCS);
   }
   
   /**
    * Sets the stream's target codepage and activates character conversion mode.
    *
    * @param    cp
    *           The target codepage for character conversion mode.
    */
   public void setConvertTarget(character cp)
   {
      if (cp == null || cp.isUnknown())
      {
         UnimplementedFeature.todo("CONVERT TARGET: confirm unknown handling is correct.");
         return;
      }
      
      setConvertTarget(cp.toStringMessage());
   }
   
   /**
    * Sets the stream's lob directory located as a subdirectory for opened current file stream directory.
    *
    * @param    lobDir
    *           The name for subdirectory to store LOB files.
    */
   public void setLobDirectory(String lobDir)
   {
      // nothing to do by default only FileStream has the real implementation
   }
   
   /**
    * Sets the stream's lob directory located as a subdirectory for opened current file stream directory.
    *
    * @param    lobDir
    *           The name for subdirectory to store LOB files.
    */
   public void setLobDirectory(character lobDir)
   {
      if (lobDir == null || lobDir.isUnknown())
      {
         UnimplementedFeature.todo("LOB-DIR: confirm unknown handling is correct.");
         return;
      }
      
      setLobDirectory(lobDir.toStringMessage());
   }
   
   /**
    * Sets the stream into echo mode which causes UI statements that read
    * data from a stream to echo that data to the current output destination.
    * If this is set on an output stream, then if the input stream also has
    * echo set to <code>true</code> then the output will be echoed, but if
    * either stream has echo as <code>false</code>, no echoing will be done.
    * Note that this only affects UI statements that are not operating
    * against the terminal.
    *
    * @param    echo
    *           The new echo mode.
    */
   public void setEcho(boolean echo)
   {
      this.echo = echo;
   }   
   
   /**
    * Gets the stream's echo mode which causes UI statements that read
    * data from a stream to echo that data to the current output destination.
    * If this is set on an output stream, then if the input stream also has
    * echo set to <code>true</code> then the output will be echoed, but if
    * either stream has echo as <code>false</code>, no echoing will be done.
    * Note that this only affects UI statements that are not operating
    * against the terminal.
    *
    * @return   The echo mode.
    */
   public boolean getEcho()
   {
      return echo;
   }   
   
   /**
    * Gets the stream's unformatted mode flag.
    *
    * @return   If <code>true</code>, all output is in a raw unformatted
    *           mode unless overridden by an explicit format string.
    */
   public boolean getUnformatted()
   {
      return unformatted;
   }   
   
   /**
    * Sets the stream into unformatted mode which disables default format
    * processing in {@link #putField}.
    *
    * @param    fmt
    *           If <code>true</code>, all output is in a raw unformatted
    *           mode unless overridden by an explicit format string.
    */
   public void setUnformatted(boolean fmt)
   {
      unformatted = fmt;
   }   
   
   /**
    * Gets the stream's control mode flag which defines whether subsequent 
    * characters written to the stream will affect line/page counters.
    * <p>
    * Warning: in Progress this is described as ensuring that the "position"
    * (presumably the write position) will not be changed when this option 
    * is enabled.  However, this really does not make any sense in the case
    * where output characters are used for control purposes.  No support is
    * provided for this case.
    *
    * @return   If <code>true</code>, all output is emitted without 
    *           modification of line/page counters.
    */
   public boolean getControl()
   {
      return control;
   }   
   
   /**
    * Sets the stream into control mode which ensures that subsequent 
    * characters written to the stream will not affect line/page counters.
    * <p>
    * Warning: in Progress this is described as ensuring that the "position"
    * (presumably the write position) will not be changed when this option 
    * is enabled.  However, this really does not make any sense in the case
    * where output characters are used for control purposes.  No support is
    * provided for this case.
    *
    * @param    mode
    *           If <code>true</code>, all output is emitted without 
    *           modification of line/page counters.
    */
   public void setControl(boolean mode)
   {
      column = 0;
      control = mode;
   }   
   
   /**
    * Gets the stream's field import/export delimiter character.
    *
    * @return   The current delimiter <code>char</code> (defaults to
    *           {@link #DELIM}. 
    */
   public char getDelimiter()
   {
      return delim;
   }   
   
   /**
    * Sets the stream's delimiter character for import/export of fields.
    *
    * @param    delim
    *           The current delimiter <code>char</code>.
    */
   public void setDelimiter(char delim)
   {
      this.delim = delim;
   }   
   
   /**
    * Sets the stream's <code>keepMessages</code> mode which causes certain UI statements that write
    * data to the default window to be suppressed when outpu is being redirected.
    * The types of UI statements are 4GL errors and warnings, and UI statements written
    * using MESSAGE.
    */
   public void setKeepMessages()
   {
      keepMessages = true;
   }   
   
   /**
    * Set stream display mode to landscape
    */
   public void setLandscape()
   {
      displayMode = PrintMode.LANDSCAPE;
   }
   
   /**
    * Set stream display mode to portrait
    */
   public void setPortrait()
   {
      displayMode = PrintMode.PORTRAIT;
   }
   
   /**
    * Returns the display mode of the stream.
    * <p>
    * Possible values: LANDSCAPE, PORTRAIT and NONE.
    *  
    * @return The display mode for this stream as one of the <code>PrintMode</code>
    *         enum values.
    */
   public PrintMode getDisplayMode()
   {
      // TODO use this in  Stream implementations for LANDSCAPE and PORTRAIT logic
      return displayMode;
   }

   /**
    * Checks this stream redirection type: named or unnamed.
    *
    * @return   <code>true</code> if this is an unnamed stream. 
    */
   public boolean isUnnamed()
   {
      return unnamed;
   }   
   
   /**
    * Marks this stream as being named or unnamed.
    * 
    * @param    unnamed
    *           If <code>true</code>, then this stream is unnamed.
    */
   public void setUnnamed(boolean unnamed)
   {
      this.unnamed = unnamed;
   }
 
   /**
    * Makes the stream UNBUFFERED.
    */
   public void setUnbuffered()
   {
      unbuffered = true;
   }

   /**
    * Determines whether the stream is UNBUFFERED.
    *
    * @return <code>true</code> if the stream is UNBUFFERED.
    */
   public boolean isUnbuffered()
   {
      return unbuffered;
   }
   
   /**
    * Sets the stream's delimiter character for import/export of fields.
    *
    * @param    delim
    *           The current delimiter <code>char</code> (if this is not
    *           a single character, only the first character will be used).
    *           If <code>null</code> or an empty string, the delimiter is
    *           reset to the default of {@link #DELIM}.
    */
   public void setDelimiter(String delim)
   {
      if (delim == null)
      {
         this.delim = DELIM;
      }
      else
      {
         this.delim = delim.length() > 0 ? delim.charAt(0) : DELIM;
      }
   }
   
   /**
    * Gets the stream's keepMessages mode which causes certain UI statements that write
    * data to the default window to be suppressed when outpu is being redirected.
    * The types of UI statements are 4GL errors and warnings, and UI statements written
    * using MESSAGE.
    *
    * @return   The keepMessages mode.
    */
   public boolean isKeepMessages()
   {
      return keepMessages;
   }   
   
   /**
    * Get the name of the stream variable used in the legacy 4GL source code.
    *
    * @return   The legacy variable name.
    */
   public String getName()
   {
      return name;
   }
   
   /**
    * Set the name of the stream variable used in the legacy 4GL source code.
    *
    * @param    name
    *           The legacy variable name.
    */
   public void setName(String name)
   {
      this.name = name;
   }

   /**
    * Sets the flush-before-put flag. The method is called to inform the stream to flush any buffered data
    * before PUT writes to the stream. This is needed to flush any external data sources connected to this
    * stream.
    *
    * @param   value
    *          The new value to set.
    */
   public void setFlushBeforePut(boolean value)
   {
      flushBeforePut = value;
   }

   /**
    * Raise an ERROR condition if the caller cannot input from the stream due
    * to the input source being closed or never having been opened.
    *
    * @return   <code>true</code> if the calling code should silently return
    *           which will occur if the condition would have been raised but
    *           wasn't due to NO-ERROR protection (silent error mode). Will
    *           return <code>false</code> if it is OK to read from the stream.
    */
   public boolean protectReads()
   {
      if (!isIn())
      {
         String msg = String.format("Attempt to read from closed stream %s",
                                    (name == null) ? "" : name);
         ErrorManager.recordOrThrowError(1386, msg);
         return true;
      }
      
      return false;
   }
   
   /**
    * Raise an ERROR condition if the caller cannot output to the stream due
    * to the output destination being closed or never having been opened.
    *
    * @return   <code>true</code> if the calling code should silently return
    *           which will occur if the condition would have been raised but
    *           wasn't due to NO-ERROR protection (silent error mode). Will
    *           return <code>false</code> if it is OK to write to the stream.
    */
   public boolean protectWrites()
   {
      if (!isOut())
      {
         if (!isSilent())
         {
            String msg = String.format("Attempt to write to closed stream %s",
                                       (name == null) ? "" : name);
            ErrorManager.recordOrThrowError(1387, msg);
         }
         return true;
      }
      
      return false;
   }

   /**
    * Check if stream is in silent mode.
    * 
    * @return  <code>false</code>, by default all streams are verbose.
    */
   public boolean isSilent()
   {
      return false;
   }
   
   /**
    * Read a character from the stream, blocking until input is received.
    * <p>
    * The exact character read from the input stream will be returned,
    * except that when <code>binary</code> mode is false, all 0x0A characters
    * (line feeds) are converted into a 0x0D (carriage return or a Progress
    * "RETURN" key).
    * <p>
    * The underlying stream subclass determines the content of the result. 
    * Byte oriented streams such as pipes or files will return a byte while
    * streams that generate keystrokes or characters may return a DBCS or
    * Unicode character.  Please see the {@link #readCh} method of the
    * subclass to obtain more information.
    *
    * @return   The character read, -1 if there is any interruption or
    *           error during processing or -2 at <code>EOF</code>.
    */
   public integer readChar()
   {
      if (protectReads())
      {
         return new integer(-1);
      }
      
      int ch = readCh();
      
      if (!binary)
      {
         if (isWindows)
         {
            if (ch == CR)
            {
               int lf = peekCh();
               if (lf == LF)
               {
                  readCh(); // truly read / 'burn' the LF
               }
            }
            else if (ch == LF)
            {
               ch = CR;
            }
         }
         else // on Linux emulation  
         {
            if (ch == LF)
            {
               ch = CR;
            }
         }
      }
      
      return new integer(ch);
   }
   
   /**
    * Read a character from the stream, pausing until input is received or
    * the requested number of seconds have elapsed.
    * <p>
    * The exact character read from the input stream will be returned,
    * except that when <code>binary</code> mode is false, all 0x0A characters
    * (line feeds) are converted into a 0x0D (carriage return or a Progress
    * "RETURN" key).
    * <p>
    * The underlying stream subclass determines the content of the result. 
    * Byte oriented streams such as pipes or files will return a byte while
    * streams that generate keystrokes or characters may return a DBCS or
    * Unicode character.  Please see the {@link #readCh} method of the
    * subclass to obtain more information.
    *
    * @param    pause
    *           The milliseconds to wait.  If this is negative, the pause is
    *           ignored and the wait is indefinite.  If 0, there is an
    *           immediate return of -1 if no character is available for
    *           reading.
    *
    * @return   The character read, -1 if the timeout occurs or if there is
    *           any interruption or error during processing or -2 at 
    *           <code>EOF</code>.
    *
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   @SuppressWarnings("static-access")
   public integer readChar(int pause)
   throws StopConditionException
   {
      if (protectReads())
         return new integer(-1);
      
      try
      {
         if (pause < 0)
         {
            // indefinite wait for a key press
            return readChar();
         }
         if (pause == 0)
         {
            if (available() > 0)
            {
               return readChar();
            }
         }
         else
         {
            // timed wait in short increments
            long millis = System.currentTimeMillis();
            long target = millis + pause;
            
            while (millis <= target)
            {
               if (available() > 0)
               {
                  return readChar();
               }
               
               // pause for a short period
               Thread.currentThread().sleep(increment);
               
               // detect how close we are to the end
               millis = System.currentTimeMillis();
            }
         }
      }
      catch (InterruptedIOException | InterruptedException iioe)
      {
         ErrorManager.handleStopException(new StopConditionException(iioe));
      }
      catch (IOException ioe)
      {
         // silently return error below
      }
      
      return new integer(-1);
   }

   /**
    * Resets the internal buffer which holds the current line being parsed
    * and the current index into that buffer.  This must be called after
    * all {@link #readField} processing has occurred for a given line.
    * <p>
    * This also resets the current delimiter to the default.
    */
   public void resetCurrentLine()
   {
      setDelimiter(null);
      currentIn = null;
      indexIn   = 0;
   }
   
   /**
    * Read the fields from the current line buffer using the previously
    * specified delimiter using {@link #setDelimiter} or the default 
    * delimiter of {@link #DELIM} before assigning the data into the given 
    * variable <code>var</code> elements in the order the data was read. The
    * number of fields read is equal to the number of elements in the array.
    * <p>
    * Parsing is done using {@link #readFieldWorker}.
    * <p>
    * If the current line buffer is empty, a new line will be read from the
    * stream.
    * <p>
    * In the following conditions, no assignment will be made:
    * <ul>
    *   <li> a field is requested after the entire line has already been
    *        processed
    *   <li> an unquoted hyphen is used as the contents of the field AND the 
    *        delimiter is NOT a hyphen (the index is still incremented past
    *        the field)
    *   <li> the first char of a field starts with a double quote but there
    *        is no closing quote (the index is incremented to the end of the
    *        buffer)
    *   <li> the <code>var</code> element is <code>null</code> (the caller
    *        wants the field to be read and the index incremented but the
    *        contents are ignored), this is equivalent to {@link #skipField}
    * </ul>
    * <p>
    * After all field reads for a given line are complete, 
    * {@link #resetCurrentLine} must be called to enable input for the
    * next line.  Otherwise, all subsequent reads will continue trying to
    * read from the current line. This will result in reading data left on
    * the current line or reading nothing if the line has already been fully 
    * read.  <code>resetCurrentLine</code> notifies the stream environment
    * that it should move to processing the next line.
    * <p>
    * This method is used to implement the field level input reading for the <code>IMPORT</code> 4GL
    * language statement.  It does not handle the line reading mode (<code>IMPORT UNFORMATTED</code>).
    *
    * @param    var
    *           The array of elements to which the results are assigned. Must
    *           not be <code>null</code>, but any given element in the array
    *           may be <code>null</code>.
    *           
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void readField(BaseDataType[] var)
   throws EndConditionException,
          ErrorConditionException
   {
      if (protectReads())
      {
         return;
      }
      
      for (BaseDataType aVar : var)
      {
         readField(aVar);
      }
   }

   public void readField(ImportFieldExpansion f)
   {
      // TODO:
   }
   
   /**
    * Read the next field from the current line buffer using the previously
    * specified delimiter using {@link #setDelimiter} or the default
    * delimiter of {@link #DELIM} before assigning the data using the given
    * database field reference <code>ref</code>.
    * <p>
    * Parsing is done using {@link #readFieldWorker}.
    * <p>
    * If the current line buffer is empty, a new line will be read from the
    * stream.
    * <p>
    * In the following conditions, no assignment will be made:
    * <ul>
    *   <li> a field is requested after the entire line has already been
    *        processed
    *   <li> an unquoted hyphen is used as the contents of the field AND the
    *        delimiter is NOT a hyphen (the index is still incremented past
    *        the field)
    *   <li> the first char of a field starts with a double quote but there
    *        is no closing quote (the index is incremented to the end of the
    *        buffer)
    *   <li> the <code>ref</code> reference is <code>null</code> (the caller
    *        wants the field to be read and the index incremented but the
    *        contents are ignored), this is equivalent to {@link #skipField}
    * </ul>
    * <p>
    * After all field reads for a given line are complete,
    * {@link #resetCurrentLine} must be called to enable input for the
    * next line.  Otherwise, all subsequent reads will continue trying to
    * read from the current line. This will result in reading data left on
    * the current line or reading nothing if the line has already been fully
    * read.  <code>resetCurrentLine</code> notifies the stream environment
    * that it should move to processing the next line.
    *
    * @param    ref
    *           The database field reference using which the result is
    *           assigned.
    *
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void readField(FieldReference ref)
   throws EndConditionException,
          ErrorConditionException
   {
      if (protectReads())
         return;
      
      if (ref == null)
      {
         // Skip field.
         readField((BaseDataType) null);
      }
      else
      {
         Integer extent = ref.getExtent();
         if (ref.getIndex() >= 0 || extent == null)
         {
            // a non-extent or indexed extent field reference
            BaseDataType var = ref.get();
            readField(var);

            if (ref.getType() == blob.class)
            {
               var.assign(PropertyMapper.preprocessBlob(new character(var)));
            }
            else if (ref.getType() == clob.class)
            {
               var.assign(PropertyMapper.preprocessClob(new character(var)));
            }
            else
            {
               ref.set(var);
            }
         }
         else
         {
            // an extent field reference, read all individual subscripts
            for (int i = 0; i < extent; i++)
            {
               readField(new FieldReference(ref, i));
            }
         }
      }
   }
   
   /**
    * Read the next field from the current line buffer using the previously
    * specified delimiter using {@link #setDelimiter} or the default 
    * delimiter of {@link #DELIM} before assigning the data into the given 
    * variable <code>var</code>.
    * <p>
    * Parsing is done using {@link #readFieldWorker}.
    * <p>
    * If the current line buffer is empty, a new line will be read from the
    * stream.
    * <p>
    * In the following conditions, no assignment will be made:
    * <ul>
    *   <li> a field is requested after the entire line has already been
    *        processed
    *   <li> an unquoted hyphen is used as the contents of the field AND the 
    *        delimiter is NOT a hyphen (the index is still incremented past
    *        the field)
    *   <li> the first char of a field starts with a double quote but there
    *        is no closing quote (the index is incremented to the end of the
    *        buffer)
    *   <li> the <code>var</code> reference is <code>null</code> (the caller
    *        wants the field to be read and the index incremented but the
    *        contents are ignored), this is equivalent to {@link #skipField}
    * </ul>
    * <p>
    * After all field reads for a given line are complete, 
    * {@link #resetCurrentLine} must be called to enable input for the
    * next line.  Otherwise, all subsequent reads will continue trying to
    * read from the current line. This will result in reading data left on
    * the current line or reading nothing if the line has already been fully 
    * read.  <code>resetCurrentLine</code> notifies the stream environment
    * that it should move to processing the next line.
    *
    * @param    var
    *           The variable to which the result is assigned.
    *           
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void readField(BaseDataType var)
   throws EndConditionException,
          ErrorConditionException
   {
      if (protectReads())
         return;
      
      Object result = readFieldWorker(var);
      if (result == UNKNOWN_VALUE)
      {
         var.setUnknown();
         return;
      }
      
      // the result must be a string or null value
      assignDatum(var, (String) result, false);
   }
   
   /**
    * Read a block of data from the stream with the block size equal to the
    * size of the binary data array in the given <code>memptr</code> variable.
    * The data will be written into the memptr starting at the first byte
    * in the array.  The read data will be the smaller of the block size or
    * the file size.  Only an error or end of file condition will cause this
    * read to terminate before the block size has been read.
    *
    * @param    var
    *           The variable to which the result is assigned.
    *           
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void readBlock(memptr var)
   throws EndConditionException,
          ErrorConditionException
   {
      // TODO: memptr may be on server-side, improve this
      
      if (protectReads())
         return;
      
      int res = readBlockRemote(var);
      
      if (res == -1)
      {
         String spec = "Error during read at index %d";
         
         // TODO: actual Progress compatible text and number are unknown
         ErrorManager.recordOrThrowError(-1, String.format(spec, 0));
         return;
      }
      else if (res == -2)
      {
         throw new EndConditionException("EOF");
      }
   }
         
   /**
    * Read the next field from the current line buffer using the previously
    * specified delimiter using {@link #setDelimiter} or the default 
    * delimiter of {@link #DELIM}.
    * <p>
    * If the current line buffer is empty, a new line will be read from the
    * stream.
    * <p>
    * In the following conditions, a <code>null</code>return value will be 
    * provided:
    * <ul>
    *   <li> a field is requested after the entire line has already been
    *        processed
    *   <li> an unquoted hyphen is used as the contents of the field AND the 
    *        delimiter is NOT a hyphen (the index is still incremented past
    *        the field)
    *   <li> the first char of a field starts with a double quote but there
    *        is no closing quote (the index is incremented to the end of the
    *        buffer)
    *   <li> the <code>var</code> parameter is <code>null</code> (the caller
    *        wants the field to be read and the index incremented but the
    *        contents are ignored)
    *   <li> an error condition has been raised and silent error mode has
    *        been specified
    * </ul>
    * <p>
    * If the ? literal is found, it will return the {@link #UNKNOWN_VALUE} instance.  Otherwise, for the
    * <code>"?"</code> string, the actual string is returned.
    * <p>
    * After all field reads for a given line are complete, 
    * {@link #resetCurrentLine} must be called to enable input for the
    * next line.  Otherwise, all subsequent reads will continue trying to
    * read from the current line. This will result in reading data left on
    * the current line or reading nothing if the line has already been fully 
    * read.  <code>resetCurrentLine</code> notifies the stream environment
    * that it should move to processing the next line.
    *
    * @param    var
    *           If <code>null</code> the next field read will be discarded
    *           instead of returned.
    *
    * @return   The string data that was read, the {@link #UNKNOWN_VALUE} or <code>null</code> (see above).
    *
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public Object readFieldWorker(BaseDataType var)
   throws EndConditionException,
          ErrorConditionException,
          StopConditionException
   {
      // load our buffer if it is empty
      if (currentIn == null)
      {
         currentIn = reload(false);
         
         // this only occurs in an unrecoverable error, silently return
         if (currentIn == null)
         {
            return null;
         }
      }
      
      // if a field is requested past the end of the current line buffer,
      // we are done
      if (indexIn >= currentIn.length())
      {
         // nothing to do
         return null;
      }
      
      // if delimiter is space then multiple sequential delimiters are ignored
      if (delim == DELIM)
      {
         while (indexIn < currentIn.length() && currentIn.charAt(indexIn) == delim)
         {
            // skip the extra delimiter
            indexIn++;

            // make sure we don't move past the end
            if (indexIn == currentIn.length())
            {
               // nothing to do
               return null;
            }
         }
      }
      
      String field = null;
      int    start = indexIn;
      int    end   = 0;
      
      // if the first character is a double quote, then we have to parse a
      // quoted string and handle escaped interior quotes (doubled up double
      // quotes)
      boolean asString = currentIn.charAt(start) == '\"';
      if (asString)
      {
         boolean force = false;
         
         // move our index past the initial quote
         indexIn++;
         
         // if we find our opening quote at the end of the input, force the
         // entry to the parsing loop below because we need to try to extend
         // our input buffer; this is more complicated because the loop
         // control structure of the parsing loop MUST drop out when the
         // index is incremented past the last character in the buffer EXCEPT
         // in this first iteration
         if (indexIn == currentIn.length())
         {
            force = true;
         }
         
         // find the end of the quoted string
         while (indexIn < currentIn.length() || force)
         {
            // shift back into normal operation
            force = false;
            
            // look for the next quote
            end = currentIn.indexOf('\"', indexIn);
            
            StringBuilder sb = new StringBuilder();
            int pos = -1;
            
            // there must always be a matching quote unless there is an
            // embedded newline
            while (end < 0)
            {
               // keep reading until we find a closing quote or the end of
               // the file, any data found is appended to our input buffer
               
               // one time init
               if (pos == -1)
               {
                  pos = indexIn;
                  sb.append(currentIn);
               }
               
               // try to extend our line
               try
               {
                  String more = reload(true);
                  
                  // this only occurs in an unrecoverable error, silently
                  // return
                  if (more == null)
                  {
                     return null;
                  }
                  
                  // extend our data and add the embedded newline that was
                  // stripped off when we read the line in
                  sb.append(NEWLINE).append(more);
                  
                  // update our state
                  currentIn = sb.toString();
                  end       = currentIn.indexOf('\"', pos);
                  pos       = currentIn.length();
               }
               
               catch (EndConditionException ece)
               {
                  // no more data on this "line", continue processing here
                  // and suppress the ECE since it should not appear until
                  // the next call that tries to read a *new* line (we are
                  // trying to extend a current line which is treated 
                  // differently in Progress); end will be a negative number
                  // already which is needed
                  break;
               }
            }
               
            // check again for the matching quote   
            if (end < 0)
            {
               // we have read to the end of the file without a matching
               // quote so we must drop all the data and return as if there
               // is nothing left on this line
               
               // increment to the end of the buffer
               indexIn = currentIn.length();
               
               // no assignment, just return (all data is ignored)
               return null;
            }
            else
            {
               // possible closing quote
               if (++end < currentIn.length())
               {
                  if (currentIn.charAt(end) == '\"')
                  {
                     // keep looping, this is just an escaped (doubled up)
                     // set of interior quotes
                     indexIn = end + 1;
                     
                     // if we find our 2nd double quote character at the end
                     // of the input, force the parsing loop to iterate since
                     // we need to try to extend our input buffer; normally
                     // the parsing loop MUST drop out when the index is
                     // incremented past the last character in the buffer
                     // EXCEPT in this special case
                     if (indexIn == currentIn.length())
                     {
                        force = true;
                     }
                  }
                  else
                  {
                     // this is the end of the string and the end of the line
                     indexIn = end;
                     
                     // exit the loop
                     break;                     
                  }
               }
               else
               {
                  // this is the end of the string and the end of the line
                  indexIn = end;
                  
                  // exit the loop
                  break;
               }
            }
         }
         
         // read the field contents 
         field = currentIn.substring(start, indexIn);
                  
         // convert to an in-memory string (drop all the extra quotes)
         field = character.fromExportString(field);
      }
      else
      {
         // the non-quoted path (which can have embedded quotes so long as
         // they are not the first char after a delimiter)
         end = currentIn.indexOf(delim, start);
         
         if (end < 0)
         {
            // no delimiter is left, just copy the rest of the line by
            // setting the index to the length of the line
            indexIn = currentIn.length();            
         }
         else
         {
            // we found the end of this field, increment the index
            indexIn = end;
         }
         
         // copy the data
         field = currentIn.substring(start, indexIn);
      }
            
      // increment the index past the next delimiter
      indexIn++;
            
      // implement 2 cases where we "skip" the field:
      // 1. null variable means the caller wants us to skip this field
      // 2. hyphen means skip this field (don't assign the var, just return)
      if (var == null || (field.equals("-") && skipLoneHyphenInput()))
      {
         // nothing to do
         return null;
      }
      
      // return the text of the read field
      return asString ? field : ("?".equals(field) ? UNKNOWN_VALUE : field);
   }
   
   /**
    * Read a complete line without regard to the normal Progress formatted field processing rules 
    * or delimiters, attempting to import the data for the specified record fields.
    *
    * @param    entries
    *           The record to import into.
    *
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void readLine(FieldReference[] entries)
   {
      // only the first field gets read... 
      if (entries.length == 0)
      {
         return;
      }
      
      readLine(entries[0]);
   }

   /**
    * Read a complete line without regard to the normal Progress formatted field processing rules 
    * or delimiters, attempting to import the data for the specified record field.
    *
    * @param    entry
    *           The field to import into.
    *
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void readLine(FieldReference entry)
   {
      character line = new character();
      readLine(line);
      entry.set(line);
   }
   
   /**
    * Read a complete line without regard to the normal Progress formatted field processing rules 
    * or delimiters, attempting to import the data for the specified legacy property.
    *
    * @param    entry
    *           The property to import into.
    *
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void readLine(PropertyReference entry)
   {
      character line = new character();
      readLine(line);
      entry.set(line);
   }
   
   /**
    * Move the index into the current line buffer past the next instance of
    * the specified delimiter using {@link #setDelimiter} or the default 
    * delimiter of {@link #DELIM} and then return.
    * <p>
    * If the current line buffer is empty, a new line will be read from the
    * stream.
    * <p>
    * After all field reads for a given line are complete, 
    * {@link #resetCurrentLine} must be called to enable input for the
    * next line.  Otherwise, all subsequent reads will continue trying to
    * read from the current line. This will result in reading data left on
    * the current line or reading nothing if the line has already been fully 
    * read.  <code>resetCurrentLine</code> notifies the stream environment
    * that it should move to processing the next line.
    * 
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void skipField()
   throws EndConditionException,
          ErrorConditionException
   {
      // same thing as a readField() except we trash the return value
      readFieldWorker(null);
   }
   
   /**
    * Read a complete line without regard to the normal Progress formatted
    * field processing rules or delimiters.
    * <p> 
    * At this time, this method is only called for implementing IMPORT UNFORMATTED.
    *
    * @param    var
    *           The <code>BaseDataType</code> variable to which the result is
    *           assigned.  This is contrary to Progress docs which state that
    *           the field must be a <code>character</code> field (the docs
    *           are incorrect).
    *
    * @throws   EndConditionException
    *           On EOF.
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void readLine(BaseDataType var)
   throws EndConditionException,
          ErrorConditionException,
          StopConditionException
   {
      if (protectReads())
      {
         return;
      }
      
      try
      {
         // for BinaryData targets, we need to read bytes and not lines
         String result = var instanceof BinaryData ? null : readNextLine(false, true);
         
         // check flag
         if (nullQuirk)
         {
            // no assignment case (happens when there is an embedded null
            // character in the data and there is no FOLLOWING line with
            // a newline before EOF)
            return;
         }
         
         if (var instanceof recid)
         {
            ((recid) var).assign(new recid(result));
         }
         else if (var instanceof integer)
         {
            // integer must come before int64 and date Otherwise the operator instanceof 
            // will match the integer instances, too
            ((integer) var).assign(new integer(result));
         }
         else if (var instanceof int64)
         {
            ((int64) var).assign(new int64(result));
         }
         else if (var instanceof decimal)
         {
            ((decimal) var).assign(new decimal(result));
         }
         else if (var instanceof datetimetz)
         {
            // datetimetz must come before datetime and date Otherwise the operator instanceof 
            // will match the datetimetz instances, too
            ((datetimetz) var).assign(new datetimetz(result));
         }
         else if (var instanceof datetime)
         {
            // datetime must come before date and date Otherwise the operator instanceof 
            // will match the datetime instances, too
            ((datetime) var).assign(new datetime(result));
         }
         else if (var instanceof date)
         {
            ((date) var).assign(new date(result));
         }
         else if (var instanceof logical)
         {
            ((logical) var).assign(new logical(result));
         }
         else if (var instanceof character)
         {
            ((character) var).assign(new character(result));
         }
         else if (var instanceof clob)
         {
            ((clob) var).assign(new clob(result));
         }
         else if (var instanceof longchar)
         {
            ((longchar) var).assign(new longchar(result));
         }
         else if (var instanceof BinaryData)
         {
            // TODO: for process streams, 4GL 'ignores' next available line fully, and from next one to that,  
            // the number of bytes as is the length of this binary data. Also, the raw variable gets set to 
            // unknown always
            
            BinaryData bin = (BinaryData) var;
            int len = (int) bin.lengthOf();
            byte[] bytes = new byte[len];
            int nextPos = 0;
            
            outerLoop:
            while (true)
            {
               byte[] nextBytes = readBytes((int) Math.min(BATCH_SIZE, len));
               if (nextBytes == null)
               {
                  if (nextPos == 0)
                  {
                     // if we are on EOF or otherwise error, raise an END
                     throw new EndConditionException("EOF");
                  }
                  break outerLoop;                  
               }

               for (int idx = 0; idx < nextBytes.length; idx++)
               {
                  if (nextPos >= bytes.length)
                  {
                     break outerLoop;
                  }
                  
                  byte nextByte = nextBytes[idx];
                  if (nextByte == 26 && isWindows && !binary)
                  {
                     setPosition(nextPos + 1);
                     // byte 26 (CTRL-Z) is interpreted as EOF only on Windows when in non-binary mode
                     break outerLoop;
                  }
                  
                  // all '\r\n' sequences are replaced by '\n' (for '!binary && isWindows' case)
                  if (nextByte == '\n' && !binary && isWindows && nextPos > 0 && bytes[nextPos - 1] == '\r')
                  {
                     // check previous byte - if is '\r', replace it with '\n'
                     bytes[nextPos - 1] = '\n';
                  }
                  else
                  {
                     bytes[nextPos] = nextByte;
                     nextPos++;
                  }               
               }
               len -= nextBytes.length;
            }
            
            // ensure only the actual read bytes are written
            if (nextPos != bytes.length)
            {
               bytes = Arrays.copyOf(bytes, nextPos);
            }
            
            // set the byte content
            if (var instanceof raw)
            {
               ((raw) var).assign(bytes);
            }
            else
            {
               // do not overlay and do not trim the target LOB or MEMPTR buffer
               ((LargeObject) var).write(false, bytes, 0, false);
            }
         }
         else if (var instanceof rowid)
         {
            ((rowid) var).assign(new rowid(result));
         }
      }
      catch (EOFException eof)
      {
         throw new EndConditionException("EOF");
      }
      catch (InterruptedIOException | InterruptedException iioe)
      {
         ErrorManager.handleStopException(new StopConditionException(iioe));
      }
      catch (IOException ioe)
      {
         // TODO: actual Progress compatible text and number are unknown
         ErrorManager.recordOrThrowError(-1, "IOException during read.");
      }
   }
   
   /**
    * Write a field to the stream using the previously specified delimiter 
    * (using {@link #setDelimiter}) or the default delimiter of 
    * {@link #DELIM} in between each field.  All output will be formatted
    * using the <code>toStringExport</code> method of the associated
    * type.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void writeField(int val)
   throws ErrorConditionException
   {
      writeField(new integer(val));
   }
   
   /**
    * Write a field to the stream using the previously specified delimiter 
    * (using {@link #setDelimiter}) or the default delimiter of 
    * {@link #DELIM} in between each field.  All output will be formatted
    * using the <code>toStringExport</code> method of the associated
    * type.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void writeField(double val)
   throws ErrorConditionException
   {
      writeField(new decimal(val));
   }
   
   /**
    * Write a field to the stream using the previously specified delimiter 
    * (using {@link #setDelimiter}) or the default delimiter of 
    * {@link #DELIM} in between each field.  All output will be formatted
    * using the <code>toStringExport</code> method of the associated
    * type.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void writeField(boolean val)
   throws ErrorConditionException
   {
      writeField(new logical(val));
   }
   
   /**
    * Write a field to the stream using the previously specified delimiter 
    * (using {@link #setDelimiter}) or the default delimiter of 
    * {@link #DELIM} in between each field.  All output will be formatted
    * using the <code>toStringExport</code> method of the associated
    * type.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The value to be output.
    *
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void writeField(String val)
   throws ErrorConditionException
   {
      writeField(new character(val));
   }
   
   /**
    * Write a field to the stream using the previously specified delimiter 
    * (using {@link #setDelimiter}) or the default delimiter of 
    * {@link #DELIM} in between each field.  All output will be formatted
    * using the <code>toStringExport</code> method of the associated
    * type.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void writeField(BaseDataType val)
   throws ErrorConditionException
   {
      // are we the first field in this output line?
      if (column > 0)
      {
         // not the first, so we write the delimiter
         append(delim);
      }
      
      // write the actual field in export "format"
      append(val.toStringExport());
   }
   
   /**
    * Write the entire block of data (as represented by the given  <code>BinaryData</code>) to the stream
    * with no delimiters, end of line insertions or other modifications.  All output will not be formatted
    * and exported using the <code>getByteArray</code> method of the associated type.
    * <p>
    * Use of this method does NOT require bracketing between
    * {@link #startOutput} and {@link #endOutput} methods and these must
    * not be called otherwise the result is undefined.
    * <p>
    * This method is called only from client-side, via the ExportField.output method.
    * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
    * 
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void writeBlock(BinaryData val)
   throws ErrorConditionException,
          StopConditionException
   {
      // TODO: memptr may be on server-side, improve this
      
      try
      {
         if (val.isUnknown())
         {
            write("?");
         }
         else
         {
            if (val.lengthOf() > 0)
            {
               // negative array sizes cause a memory access violation in 4GL
               // but here we just avoid the failure
               
               // TODO: test zero length array sizes

               write(val.getByteArray());
            }
         }
      }
      
      catch (InterruptedIOException iioe)
      {
         ErrorManager.handleStopException(new StopConditionException(iioe));
      }
      
      catch (IOException ioe)
      {
         ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
      }
   }
   
   /**
    * Write the entire block of data (as represented by the given <code>blob</code>) to the stream with no
    * delimiters, end of line insertions or other modifications.  The data will be exported into separate
    * file within subdirectory defined in LOB-DIR option or if not defined in current base directory.
    * <p>
    * Use of this method does NOT require bracketing between {@link #startOutput} and {@link #endOutput}
    * methods and these must not be called otherwise the result is undefined.
    * <p>
    * This method is called only from client-side, via the ExportField.output method.
    * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
    * 
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void writeBlob(blob val)
   throws ErrorConditionException,
          StopConditionException
   {
      writeBlock(val);
   }

   /**
    * Write the entire block of data (as represented by the given <code>clob</code>) to the stream with no
    * delimiters, end of line insertions or other modifications.  The data will be exported into separate
    * file within subdirectory defined in LOB-DIR option or if not defined in current base directory.
    * <p>
    * Use of this method does NOT require bracketing between {@link #startOutput} and {@link #endOutput}
    * methods and these must not be called otherwise the result is undefined.
    * <p>
    * This method is called only from client-side, via the ExportField.output method.
    * WARNING: any other usage from server-side needs to be reviewed, as it may be expensive.
    * 
    * @param    val
    *           The value to be output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void writeClob(clob val)
   throws ErrorConditionException,
          StopConditionException
   {
      writeField(val);
   }
   
   /**
    * Inserts a page break if the stream is paged and if the next write
    * position is not on the first line of a new page.  This method
    * internally implements a {@link #startOutput} and {@link #endOutput}. 
    * This means that this method should never be called while there is
    * a current <code>startOutput</code> is in effect.
    * <p>
    * If page footers are in use, any partial page will be extended to the
    * full size before the footers are output.
    * <p>
    * This method does nothing if the stream is in <code>control</code> mode.
    * 
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void advancePage()
   throws ErrorConditionException
   {
      if (protectWrites())
         return;
      
      if (!control && rawGetPageSize() > 0)
      {
         // TODO: it is unknown if an explicit NL should be appended here
         endOutput();
         processAdvanceNotifications();

         if (rawGetNextLineNum() > 1)
         {
            startOutput(0);
            
            try
            {
               extendPage();
            }
            catch (IOException ioe)
            {
               ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
            }
            
            writePageBreak();
            endOutput();
         }
      }
   }
   
   /**
    * This method does nothing if the stream is in <code>control</code> mode.
    */
   public void flush()
   {
      if (flushListeners != null)
      {
         Iterator<FlushListener> list = flushListeners.iterator();
         
         while (list.hasNext())
         {
            FlushListener fl = list.next();
            fl.notifyFlush(this);
            list.remove();
         }      
      }
   }
   
   /**
    * Create the buffer to gather all output of {@link #putField},
    * {@link #putSpace}, {@link #putLineEnd} and {@link #writeField} methods.
    * When filled, the buffer must be written to the stream using
    * {@link #endOutput}.
    * <p>
    * There must always be an intervening <code>endOutput</code> between 
    * every two calls to this method.  Do not try to nest calls to this
    * method.
    * <p>
    * This method does nothing if the stream is in <code>control</code> mode.
    * 
    * @param    newColumn
    *           Initial value for column counter.
    */
   public void startOutput(int newColumn)
   {
      processHeaders();
      
      if (!control)
      {
         column = newColumn;
         
         if (active)
            return;
         
         currentOut = new StringBuilder();
         active     = true;
      }
   }
   
   /**
    * Write the current output buffer to the stream. This flushes all 
    * buffered output of the {@link #putField}, {@link #putSpace},
    * {@link #putLineEnd} and {@link #writeField} methods to the stream.
    * This method also resets the current delimiter to the default.
    * <p>
    * This method does nothing if the stream is in <code>control</code> mode.
    * 
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void endOutput()
   throws ErrorConditionException,
          StopConditionException
   {
      endOutput(false);
   }

   /**
    * Write the current output buffer to the stream. This flushes all 
    * buffered output of the {@link #putField}, {@link #putSpace},
    * {@link #putLineEnd} and {@link #writeField} methods to the stream.
    * This method also resets the current delimiter to the default.
    * <p>
    * This method does nothing if the stream is in <code>control</code> mode.
    * 
    * @param    appendNewLine
    *           Automatically add new line at the end of output.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void endOutput(boolean appendNewLine)
   throws ErrorConditionException,
          StopConditionException
   {
      if (!control)
      {
         if (!active)
            return;
         
         try
         {
            if (!trailingNL && appendNewLine)
            {
               writeNewLine();
               rawIncrementNextLineNum();
               setTrailingNL(true);
            }
            
            String output = currentOut.toString();

            write(output);

            // we will count only if the written char is not a page-break
            if (!"\f".equals(output))
               lastPageWithOutput = rawGetPageNum();
         }
         
         catch (InterruptedIOException iioe)
         {
            ErrorManager.handleStopException(new StopConditionException(iioe));
         }
         
         catch (IOException ioe)
         {
            ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
         }
         
         finally
         {
            // if the put worker still in progress we need to just clean the buffer instead of
            // removing it leaving them active and ready for incoming new data
            synchronized (outActivityLock)
            {
               if (putWorkerActive)
               {
                  currentOut.setLength(0);
               }
               else
               {
                  currentOut = null;
                  active     = false;
               }
            }
         }
      }
   }
   
   /**
    * Notifies the stream that an empty drawing has occurred.
    * This has the effect of kicking off the page headers if at the page
    * boundary. 
    */
   public void pulseOutput()
   {
      if (!control && rawGetPageSize() > 0 && rawGetNextLineNum() == 1)
      {
         processHeaders();
      }
   }

   /**
    * Append a platform-specific line-end to the current output buffer if
    * the output is not already positioned at the beginning of a line OR
    * if in <code>control</code> mode.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    * <p>
    * This method is built to duplicate the behavior of the SKIP option with
    * the put statement.
    * 
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putLineEnd()
   throws ErrorConditionException
   {
      putLineEnd(0);
   }
   
   /**
    * Append the specified number of platform-specific line-ends to the
    * current output buffer.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param   num
    *          The number of blank lines to follow the current line (which
    *          means that this number of line-ends + 1 must be appended).
    *          If this is negative, no output is generated. If 0, 1 line-end
    *          is output if (and only if) the current output position is not
    *          already at the beginning of the line.  If 1, then 2 line-ends
    *          are output and so on.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putLineEnd(int num)
   throws ErrorConditionException
   {
      if (num < 0)
      {
         // ignore the call
         return;
      }
      else
      {
         // compute the number of remaining empty rows
         // remaining-empty-rows = PAGE-SIZE - LINE-COUNTER + 1 - footerSize 
         int pageSz = rawGetPageSize();
         int nextLineNum = rawGetNextLineNum();
         int remainingRows = pageSz - nextLineNum  + 1 - footerSize;
         
         // do not adjust number of rows when page size is not set
         if (pageSz > 0 && num > 0 && num > remainingRows)
         {
            num = remainingRows;
         }
         
         // add NL if there was unfinished output
         processLineEnd(true, true);
         for (int i = 0; i < num; i++)
         {
            processLineEnd(false, true);
         }
      }
   }
   
   /**
    * Append a space to the current output buffer.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putSpace()
   throws ErrorConditionException
   {
      append(' ');
   }
   
   /**
    * Append the given number of spaces to the current output buffer.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    num
    *           The number of spaces to append.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putSpace(int num)
   throws ErrorConditionException
   {
      for (int i = 0; i < num; i++)
      {
         append(' ');
      }      
   }
   
   /**
    * Append a null character '\0' to the current output buffer.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putNull()
   throws ErrorConditionException
   {
      append('\0');
   }
   
   /**
    * Append the given number of null characters '\0' to the current output
    * buffer.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    num
    *           The number of nulls to append.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putNull(int num)
   throws ErrorConditionException
   {
      for (int i = 0; i < num; i++)
      {
         append('\0');
      }      
   }
   
   /**
    * Append the <code>int</code> value to the output buffer with the default
    * format string of '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.    
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(int val)
   throws ErrorConditionException
   {
      putField(val, null);
   }
   
   /**
    * Append the <code>int</code> value to the output buffer with the given
    * format string at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' will be used.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(int val, String fmt)
   throws ErrorConditionException
   {
      putField(new integer(val), fmt);
   }
   
   /**
    * Append the <code>int</code> value to the output buffer with the default
    * format string of '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(int val, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col);
   }
   
   /**
    * Append the <code>int</code> value to the output buffer with the given
    * format string at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(int val, String fmt, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(new integer(val), fmt, starts, col);
   }
   
   /**
    * Append the <code>int</code> value to the output buffer with the default
    * format string of '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(int val, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col.intValue());
   }
   
   /**
    * Append the <code>int</code> value to the output buffer with the given
    * format string at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           '-&gt;,&gt;&gt;&gt;,&gt;&gt;9' will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(int val, String fmt, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(new integer(val), fmt, starts, col.intValue());
   }
   
   /**
    * Append the <code>double</code> value to the output buffer with the
    * default format string of '-&gt;&gt;,&gt;&gt;9.99' at the current output column. 
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.    
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(double val)
   throws ErrorConditionException
   {
      putField(val, null);
   }
   
   /**
    * Append the <code>double</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           '-&gt;&gt;,&gt;&gt;9.99' will be used.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(double val, String fmt)
   throws ErrorConditionException
   {
      putField(new decimal(val), fmt);
   }
   
   /**
    * Append the <code>double</code> value to the output buffer with the
    * default format string of '-&gt;&gt;,&gt;&gt;9.99' at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(double val, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col);
   }
   
   /**
    * Append the <code>double</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           '-&gt;&gt;,&gt;&gt;9.99' will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(double val, String fmt, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(new decimal(val), fmt, starts, col);
   }
   
   /**
    * Append the <code>double</code> value to the output buffer with the
    * default format string of '-&gt;&gt;,&gt;&gt;9.99' at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(double val, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col.intValue());
   }
   
   /**
    * Append the <code>double</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Note that this format string works as the display form, which pads on
    * the left to the full size of the format string.  This is different from
    * the normal operation of <code>toString</code>.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           '-&gt;&gt;,&gt;&gt;9.99' will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(double     val,
                        String     fmt,
                        boolean    starts,
                        NumberType col)
   throws ErrorConditionException
   {
      putField(new decimal(val), fmt, starts, col.intValue());
   }
   
   /**
    * Append the <code>boolean</code> value to the output buffer with the
    * default format string of 'yes/no' at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(boolean val)
   throws ErrorConditionException
   {
      putField(val, null);
   }
   
   /**
    * Append the <code>boolean</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           'yes/no' will be used.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(boolean val, String fmt)
   throws ErrorConditionException
   {
      putField(new logical(val), fmt);
   }
   
   /**
    * Append the <code>boolean</code> value to the output buffer with the
    * default format string of 'yes/no' at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(boolean val, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col);
   }
   
   /**
    * Append the <code>boolean</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           'yes/no' will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(boolean val, String fmt, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(new logical(val), fmt, starts, col);
   }
   
   /**
    * Append the <code>boolean</code> value to the output buffer with the
    * default format string of 'yes/no' at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(boolean val, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col.intValue());
   }
   
   /**
    * Append the <code>boolean</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           'yes/no' will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(boolean    val,
                        String     fmt,
                        boolean    starts,
                        NumberType col)
   throws ErrorConditionException
   {
      putField(new logical(val), fmt, starts, col.intValue());
   }
   
   /**
    * Append the <code>String</code> value to the output buffer with a 
    * format string of 'x(N)' (where N is the length of the value) at the 
    * current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(String val)
   throws ErrorConditionException
   {
      putField(val, null);
   }
   
   /**
    * Append the <code>String</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           'x(N)' will be used (where N is the length of the value).
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(String val, String fmt)
   throws ErrorConditionException
   {
      putField(new character(val), generateStringFormat(fmt, val.length()));
   }
   
   /**
    * Append the <code>String</code> value to the output buffer with the
    * default format string of 'x(N)' will be used (where N is the length of
    * the value) at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(String val, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col);
   }
   
   /**
    * Append the <code>String</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           'x(N)' will be used (where N is the length of the value) will
    *           be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(String val, String fmt, boolean starts, int col)
   throws ErrorConditionException
   {
      fmt = generateStringFormat(fmt, val.length());
      putField(new character(val), fmt, starts, col);
   }
   
   /**
    * Append the <code>String</code> value to the output buffer with the
    * default format string of 'x(N)' will be used (where N is the length of
    * the value) at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(String val, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col.intValue());
   }
   
   /**
    * Append the <code>String</code> value to the output buffer with the
    * given format string at the current output column.
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           'x(N)' will be used (where N is the length of the value) will
    *           be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(String     val,
                        String     fmt,
                        boolean    starts,
                        NumberType col)
   throws ErrorConditionException
   {
      fmt = generateStringFormat(fmt, val.length());
      putField(new character(val), fmt, starts, col.intValue());
   }
   
   /**
    * Append the value to the output buffer with the default type-specific 
    * format string at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(BaseDataType val)
   throws ErrorConditionException
   {
      putField(val, null);
   }
   
   /**
    * Append the value to the output buffer with the given type-specific 
    * format string at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           type-specific format will be used.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(BaseDataType val, String fmt)
   throws ErrorConditionException
   {    
      if (control && val instanceof raw)
      {
         try
         {
            write(((raw)val).getByteArray());
         }
         
         catch (InterruptedIOException iioe)
         {
            ErrorManager.handleStopException(new StopConditionException(iioe));
         }
         
         catch (IOException ioe)
         {
            ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
         }
         return;
      }
      append(generateFormattedText(val, fmt));
   }
   
   /**
    * Append the value to the output buffer with the default type-specific 
    * format string at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(BaseDataType val, boolean starts, int col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col);
   }
   
   /**
    * Append the value to the output buffer with the default type-specific 
    * format string at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(BaseDataType val, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(val, null, starts, col.intValue());
   }
   
   /**
    * Append the value to the output buffer with the given type-specific 
    * format string at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           type-specific format will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(BaseDataType val, String fmt, boolean starts, NumberType col)
   throws ErrorConditionException
   {
      putField(val, fmt, starts, col.intValue());
   }      
      
   /**
    * Append the value to the output buffer with the given type-specific 
    * format string at the current output column. 
    * <p>
    * Use of this method generates output to a buffer which must first be
    * initialized by {@link #startOutput}.  Once initialized, any number of
    * calls to this method (or related output methods) may be done and all
    * output will be gathered into the output buffer.  To write this buffer
    * to the stream, call {@link #endOutput}. Calls to this method must never
    * be done outside of a bracketed pair of matching <code>startOutput</code>
    * and <code>endOutput</code> calls otherwise the results will be
    * undefined.
    *
    * @param    val
    *           The data to be output.
    * @param    fmt
    *           The format string to use.  If <code>null</code>, the default
    *           type-specific format will be used.
    * @param    starts
    *           If <code>true</code>, the output will be placed starting at
    *           the given column (or on the next line at that column, if the
    *           given column is already in use on the current line). If
    *           <code>false</code>, the output will be placed so that it
    *           ends at the given column (or on the next line at that column,
    *           if the calculated start column is already in use on the
    *           current line).
    * @param    col
    *           The start or end column (which it is depends on the previous 
    *           parameter's value). This is used as a 1-based index into the
    *           current output line.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putField(BaseDataType val, String fmt, boolean starts, int col)
   throws ErrorConditionException
   {
      // obtain the text to output, based on type-specific formatting
      String text = generateFormattedText(val, fmt);
      int    len  = text.length();
      
      // convert the 1-based index into the 0-based index that we need
      col--;
      
      // calculate starting position if needed
      if (!starts)
      {
         col = (col + 1) - len;
      }
      
      // do we need to move to the next line?
      if (col < column)
      {
         processLineEnd(false, true);

         // we must check for page-break after we output this NL
         checkForPageBreak();
      }
      
      int pad = col - column;
      
      // pad if necessary
      if (pad > 0)
      {
         putSpace(pad);
      }
      
      // output the text
      append(text);
   }
   
   /**
    * Output a list of fields (without buffering) to the stream. 
    *
    * @param    data
    *           The list of fields to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putControl(FieldEntry[] data)
   throws ErrorConditionException
   {
      if (protectWrites())
      {
         return;
      }
      
      endOutput();
      
      // this sets this value on the server side (in case it is needed)
      setControl(true);
      
      try
      {
         putWorker(data, MODE_PUT_CONTROL, (char) -1);
      }
      finally
      {
         // this resets this value on the server side
         setControl(false);
      }
   }
   
   /**
    * Output a list of fields with default formatting to the stream. 
    *
    * @param    data
    *           The list of fields to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putUnformatted(FieldEntry[] data)
   throws ErrorConditionException
   {
      if (protectWrites())
      {
         return;
      }
      
      putWorker(data, MODE_PUT_UNFORMATTED, (char) -1);
   }
   
   /**
    * Output a list of fields with specified formatting to the stream. 
    *
    * @param    data
    *           The list of fields to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void put(FieldEntry[] data)
   throws ErrorConditionException
   {
      if (protectWrites())
      {
         return;
      }
      
      // by default no automatic line feed should be performed
      putWorker(data, MODE_PUT_NORMAL, (char) -1);
   }
   
   /**
    * Output a list of fields with export formatting to the stream. 
    *
    * @param    data
    *           The list of fields to be output.
    *          
    * @throws   ErrorConditionException
    *           On any error during I/O.
    * @throws   StopConditionException
    *           If any interruption occurs during I/O processing.
    */
   public void export(FieldEntry[] data)
   throws ErrorConditionException,
          StopConditionException
   {
      if (protectWrites())
      {
         return;
      }
      
      putWorker(data, MODE_EXPORT, delim);
   }
   
   /**
    * Writes the given data into a buffer appending a newline to the end.
    * This buffered data will be pushed out to the stream with a subsequent
    * call to {@link #flushControlled}.
    *
    * @param    txt
    *           The data to write.  May be <code>null</code> in which case
    *           nothing is written to the buffer. If this is the empty
    *           string, only a newline is written into the buffer.
    */
   public void writeControlled(String txt)
   {
      boolean isNeedNL = column != 0 && !inHeaders && !trailingWithoutPutNL;
      
      if (txt == null)
      {
         //nothing to do
         return;
      }
      
      if (buffer == null)
      {
         buffer = new ArrayList<>();
      }
      StringBuilder sb = new StringBuilder(txt.length() + 2);
      if (isNeedNL) 
      {
         trailingWithoutPutNL = true;
         sb.append(NEWLINE);
      }
      sb.append(txt).append(NEWLINE);
      
      buffer.add(sb.toString());
   }
   
   /**
    * Supports writing form feeds and intermediate headers.
    * Required for a proper implementation of redirected editors.
    * Writes the given data into a buffer and immediately flushes the buffer
    * to the stream. Advances the page number and adjusts the line number
    * depending on the height of the headers. Suppresses the form feed output
    * for the nearest page break. 
    *
    * @param    txt
    *           The data to write. The form feed is expected to be the first 
    *           character, followed by optional frame header lines, although 
    *           it is not checked
    * @param    lines
    *           number of header lines in the passed text. required for
    *           adjusting the line counter
    */
   public void writeControlledSpecial(String txt, int lines)
   {
      if (buffer == null)
      {
         buffer = new ArrayList<>();
      }
      
      buffer.add(txt);
      
      boolean savecont = contiguous;
      contiguous = true;
      try
      {
         flushControlled(true);
      }
      catch (IOException ioe)
      {
      }
      contiguous = savecont;
      
      // maintain paging state
      rawIncrementPageNum();
      rawResetNextLineNum();
      for (int i = 0; i < lines; i++)
      {
         rawIncrementNextLineNum();
      }
      
      setTrailingNL(true);
      suppressFormFeed = true;
   }
   
   /**
    * Writes the data previously buffered (via {@link #writeControlled}) to
    * the stream.  This method maintains all needed state for proper paging
    * processing and it will output report headers or footers as needed.
    *
    * @param    bypass
    *           <code>true</code> if header processing must be bypassed when
    *           starting to flush the output.
    *
    * @return   The next line number AFTER all buffered lines are output but
    *           BEFORE any final page break processing has occurred. This can
    *           be compared with the next line number as queried after this
    *           method returns to determine if a page break was output or
    *           not (among other uses). If there was no output, then -1 will
    *           be returned.
    *
    * @throws   IOException 
    *           if an I/O error occurs. 
    */
   public int flushControlled(boolean bypass)
   throws IOException
   {
      int rc = -1;
      
      if (buffer == null || buffer.size() == 0) 
      {
         //nothing to do
         return rc;
      }
      
      ArrayList<String> buf  = buffer;
      Iterator<String>  list = buffer.iterator();
      flushLevel++;
      
      // "clear" the buffer so that we can recursively call in to fill the buffer again
      buffer = null;
      
      try
      {
         // only output if this is the top-level entry to this method (this
         // method has a form of indirect recursion)
         if (flushLevel == 1)
         {
            // first flush the internal buffer if any
            endOutput();

            // if we are NOT on the first line of the page AND the expected
            // output will cross a page boundary AND we are not in contiguous
            // mode then handle the form feed, page state and footers now
            // (causes recursion)

            // WARNING: This protection is needed in the case where we are
            //          outputting exactly as many lines as there are 
            //          left on the page.  Progress does NOT trigger the
            //          page break in that scenario.

            int top            = rawGetNextLineNum();
            int sz             = buf.size();
            int remainingLines = 1;
            int pageSz         = rawGetPageSize();
            
            if (pageSz > 0) 
            { 
               remainingLines = pageSz - top + 1 - footerSize;
            }
            
            if (!doneOut             &&
                top > 1              &&
                sz  > 1              &&
                sz  > remainingLines &&
                !contiguous          && 
                needsPageBreak(sz))
            {
               // fill in a partial page with blanks before processing any footer or form feed
               extendPage();
               writePageBreak();
               
               // if the page-break was written, then page headers must be output
               bypass = false;
            }
            
            if (!bypass)
            {
               // if we are at the beginning of the page, output the headers (causes recursion)
               processHeaders();
            }
         }
         
         // process our buffered output (strangely, Progress always groups the
         // entire buffered output together, even if it will cause a longer or
         // shorter page size than specified in the stream definition --> this
         // is why the header/footer processing is outside of this loop)
         while (list.hasNext())
         {
            String line = list.next();
            
            write(line);
            rawIncrementNextLineNum();
            setTrailingNL(true);
         }
         
         // get return value after the buffer is processed but before the
         // possible ending page break
         rc = rawGetNextLineNum();
         
         // if we are AT or PAST the end of the page, handle the form feed,
         // page state and footers (causes recursion)
         if (flushLevel == 1 && !doneOut && !contiguous && needsPageBreak())
         {
            writePageBreak();
         }
      }
      finally
      {
         if (flushLevel == 1)
         {
            contiguous = false;
         }
         
         flushLevel--;
      }
      
      return rc;
   }
   
   /**
    * Detects if headers need to be output.
    *
    * @return   <code>true</code> if header processing is not in process and
    *           the stream has registered page headers and the next line
    *           number is 1 (we have no output on the current page).
    */
   public boolean needsHeadersOutput()
   {
      return (!inHeaders && headers != null && rawGetNextLineNum() == 1);
   }
   
   /**
    * Detects if output is being generated on a new page and renders the
    * header(s) if needed.
    */
   public void processHeaders()
   {
      // avoid recursion
      if (inHeaders)
         return;
      
      // if starting output on a new page
      if (needsHeadersOutput())
      {
         processHeadersInt();
      }
   }

   /**
    * Detects if output is being generated on a new page and renders the
    * header(s) if needed.
    */
   public void processHeadersUncond()
   {
      try
      {
         flushControlled(true);
      }
      catch (IOException ioe)
      {
      }

      processHeadersInt();

      suppressFormFeed = false;
   }

   /**
    * Detects if output is being generated on a new page and renders the
    * header(s) if needed.
    */
   public void processHeadersInt()
   {
      inHeaders = true;
   
      boolean old = false;
      
      if (flushLevel == 0)
      {
         // preserve some flags that will possibly be reset after the call
         // to render headers since flushing resets the flag (not to
         // mention that we force the flag value too)
         old = contiguous;
         
         // we need contiguous mode to avoid triggering a page break in
         // the end of flushing since page sizes that are smaller than the
         // header size will detect that a page break is needed
         contiguous = true;
      }
      
      // output headers
      processRendering(headers, false);
      
      // restore previous flag value
      if (flushLevel == 0)
      {
         contiguous = old;
      }
   
      inHeaders = false;
   }

   /**
    * Output any registered footers in a LIFO order.
    */
   public void processFooters()
   {      
      Set<RepeatedOutput> out = footers;
      
      // add the footers in a LIFO order. 
      // TODO: performance could be improved by removing intermediary ArrayList container
      if (out.size() > 1)
      {   
         List<RepeatedOutput> l = new ArrayList<>();
         
         for (RepeatedOutput anOut : out)
         {
            l.add(0, anOut);
         }
         
         out = new LinkedHashSet<>(l);
      }

      processRendering(out, true);
   }

   /**
    * Checks if this is a paged stream and that it has at least 1 header or
    * footer registered.
    *
    * @return   <code>true</code> if this is a paged stream with at least
    *           1 header or footer.
    */
   public boolean hasPageHeadersFooters()
   {
      boolean hdrs = (headers != null && !headers.isEmpty());
      boolean ftrs = (footers != null && !footers.isEmpty());
      
      return (rawGetPageSize() > 0 && (hdrs || ftrs));
   }
   
   /**
    * Adds a listener to the list of those objects that will get a
    * notification at the top of every new page (except for the first
    * page).
    *
    * @param    pl
    *           The object to receive a notification whenever a page break is
    *           written to this stream.
    */
   public void registerPageBreakListener(PageBreakListener pl)
   {
      if (pageListeners == null)
      {
         pageListeners = new WeakHashMap<>();
      }
      
      pageListeners.put(pl, null);
   }
   
   /**
    * Adds a listener to the list of those objects that will get a
    * notification when this stream is about to be closed.
    *
    * @param    cl
    *           The object to receive a notification
    */
   public void registerCloseListener(CloseListener cl)
   {
      if (closeListeners == null)
      {
         closeListeners = new WeakHashMap<>();
      }
      
      closeListeners.put(cl, null);
   }
   
   /**
    * Adds a listener to the list of those objects that will get a
    * notification when this stream is about to be flushed.
    *
    * @param    fl
    *           The object to receive a notification
    */
   public void registerFlushListener(FlushListener fl)
   {
      if (flushListeners == null)
      {
         flushListeners = new LinkedHashSet<>();
      }
      
      flushListeners.add(fl);
   }
   
   /**
    * Adds a listener to the list of those objects that will get a
    * notification when this stream gets page advance order.
    *
    * @param    al
    *           The object to receive a notification
    */
   public void registerAdvanceListener(AdvanceListener al)
   {
      if (advanceListeners == null)
      {
         advanceListeners = new LinkedHashSet<>();
      }
      
      advanceListeners.add(al);
   }
   
   /**
    * Adds a header to the list of those headers that will be output at the
    * top of every new page.
    *
    * @param    head
    *           The header to add.
    */
   public void registerHeader(RepeatedOutput head)
   {
      if (rawGetPageSize() == 0)
      {
         // not paged so no headers are needed
         return;
      }
      
      if (headers == null)
      {
         headers = new LinkedHashSet<>();
      }
      
      if (headers.add(head))   
      {
         headerSize += head.height();
      }
   }
      
   /**
    * Adds a footer to the list of those footers that will be output at the
    * bottom of every page (even a last partial page).
    *
    * @param    foot
    *           The footer to add.
    */
   public void registerFooter(RepeatedOutput foot)
   {
      if (rawGetPageSize() == 0)
      {
         // not paged so no footers are needed
         return;
      }
      
      if (footers == null)
      {
         footers = new LinkedHashSet<>();
      }
      
      if (footers.add(foot))
      {
         footerSize += foot.height();
      }
   }
      
   /**
    * Removes a header from the list of those headers that will be output at the
    * top of every new page.
    *
    * @param    head
    *           The header to remove.
    */
   public void deregisterHeader(RepeatedOutput head)
   {
      if (headers == null)
      {
         return;
      }
      
      if (headers.remove(head))
      {
         headerSize -= head.height();
      }
      
      if (headers.isEmpty())
      {
         headers = null;
         headerSize = 0;
      }      
   }
      
   /**
    * Removes a footer from the list of those footers that will be output at
    * the bottom of every page (even a last partial page).
    *
    * @param    foot
    *           The footer to remove.
    */
   public void deregisterFooter(RepeatedOutput foot)
   {
      if (footers == null)
      {
         return;
      }
      
      if (footers.remove(foot))
      {
         footerSize -= foot.height();
      }
      
      if (footers.isEmpty())
      {
         footers = null;
         footerSize = 0;
      }
   }
      
   /**
    * Obtains the current read and/or write position in the stream, if the stream type supports 
    * such a query.
    * <p>
    * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They 
    * are mapped to the underlying Java platform that can handle streams with offsets that are 
    * based on a 64-bit <code>long</code>.
    *
    * @return  The current offset into the stream or the <code>unknown value</code> if the 
    *          stream type doesn't support a stream position.
    */
   public int64 getPosition()
   {
      long pos = 0;
      
      try
      {
         pos = getPos();
      }
      catch (Exception exc)
      {
         // return the unknown value because the stream doesn't support querying the seek 
         // position (UnsupportedOperationException) or there was an IOException
         pos = -1;
      }
      
      // return the queried position
      return (pos == -1) ? new int64() : new int64(pos);
   }
   
   /**
    * Sets the current read and/or write position in the stream, if the stream type supports the 
    * operation.
    * <p>
    * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They 
    * are mapped to the underlying Java platform that can handle streams with offsets that are 
    * based on a 64-bit <code>long</code>.
    * <p>
    * A negative offset is ignored (no action is taken).
    * <p>
    * If the specified offset is larger than the length of the stream, in
    * Progress the seek position will report as the number requested, but if
    * no subsequent writes occur to the file, the file size is left the same
    * as it was before setting the position.  If writes do occur, all writes
    * occur at the new position. In this case, the resulting file will be
    * sized based on the requested offset + the number of bytes written.
    *
    * @param    newpos
    *           The new offset.  If this is the <code>unknown value</code>,
    *           the set operation will silently return without any change.
    */
   public void setPosition(int64 newpos)
   {
      if (newpos.isUnknown())
      {
         // nothing to do
         return;
      }
      
      setPosition(newpos.longValue());
   }
   
   /**
    * Sets the current read and/or write position in the stream, if the
    * stream type supports the operation.
    * <p>
    * Starting with OpenEdge Release 10.1B, ABL includes a 64-bit integer data type. They 
    * are mapped to the underlying Java platform that can handle streams with offsets that are 
    * based on a 64-bit <code>long</code>.
    * <p>
    * A negative offset is ignored (no action is taken).
    * <p>
    * If the specified offset is larger than the length of the stream, in
    * Progress the seek position will report as the number requested, but if
    * no subsequent writes occur to the file, the file size is left the same
    * as it was before setting the position.  If writes do occur, all writes
    * occur at the new position. In this case, the resulting file will be
    * sized based on the requested offset + the number of bytes written.
    *
    * @param    newpos
    *           The new offset.  If this is the <code>unknown value</code>,
    *           the set operation will silently return without any change.
    */
   public void setPosition(long newpos)
   {
      if (newpos < 0)
      {
         // nothing to do
         return;
      }
            
      try
      {
         setPos(newpos);
      }
      catch (Exception exc)
      {
         // silently fail on UnsupportedOperationException or IOException
      }
   }
   
   /**
    * Set the current stream's read/write position to the end of the file
    * such that future writes are appending to the file.
    */
   public void setAppend()
   {
      try
      {
         setPos(getLen());
      }
      catch (Exception exc)
      {
         // silently fail on UnsupportedOperationException or IOException
      }
   }
   
   /**
    * The length of the stream in bytes.  This does not work for "streams"
    * that require sequential access.
    *
    * @return   The length of the stream or the <code>unknown value</code>
    *           if the underlying stream does not support this operation.
    */
   public int64 getLength()
   {
      long len = 0;
      
      try
      {
         len = getLen();
      }
      catch (Exception exc)
      {
         // return the unknown value because the stream doesn't support querying length
         // (UnsupportedOperationException) or there was an IOException
         return new int64();
      }
      
      // return the queried length
      return new int64(len);
   }
   
   /**
    * Processes the notification that the scope in which the stream was 
    * instantiated is ending and the object's reference will be lost after
    * this method is called.  This provides a natural and standard mechanism
    * to cleanup any resources that may need to be closed explicitly such
    * as operating system files or sockets.
    */
   public void finished()
   {
      // flush collected data if any
      endOutput();
      close();
   }
   
   /**
    * Provides a notification that the external program scope in which the object is registered is
    * is being deleted and the object's reference will be lost after this method is called.
    * <p>
    * This is a no-op for streams.
    */
   @Override
   public void deleted()
   {
      // no-op
   }
   
   /**
    * Processes the iterate notification provided by the {@link
    * TransactionManager}.  This implementation does nothing.
    */
   public void iterate()
   {
   }
   
   /**
    * Processes the retry notification provided by the {@link
    * TransactionManager}.  This implementation does nothing.
    */
   public void retry()
   {
   }
   
   /**
    * Generate the default format string of 'x(N)' will be used (where N is
    * the length of the value) IF AND ONLY IF the given input format string 
    * is <code>null</code>.
    *
    * @param    fmt
    *           The format string (return unchanged if not <code>null</code>)
    *           of <code>null</code> to trigger the generation of a default
    *           string-length format.
    * @param    len
    *           Fixed string value length. 
    *
    * @return   The input format string or a default string-length format.
    */
   public static String generateStringFormat(String fmt, int len)
   {
      if (fmt == null)
      {
         fmt = "x(" + len + ')';
      }
      
      return fmt;
   }

   /**
    * The number of bytes available to be immediately read without blocking.
    *
    * @return   The number of available bytes.
    * 
    * @throws   IOException
    *           if an I/O error occurs. 
    */
   public abstract long available()
   throws IOException;
   
   /**
    * The 0-based offset into the stream at which the next read or write will 
    * occur.  This does not work for "streams" that require sequential
    * access.
    *
    * @return   The current position in the stream.
    * 
    * @throws   UnsupportedOperationException 
    *           If the requested operation is not supported.
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public abstract long getPos()
   throws UnsupportedOperationException,   
          IOException;
   
   /**
    * Moves the current read/write position to the specified absolute 0-based
    * offset.  A negative offset is ignored (no action is taken).  If the   
    * specified offset is larger than the length of the stream, the seek
    * position will report as the number requested, but if no subsequent
    * writes occur to the file, the file is truncated to 0 bytes.  If writes
    * do occur, all writes occur at byte 0 BUT the file actually is of a size
    * that is the requested offset + the number of bytes written!
    * <p>
    * This does not work for "streams" that require sequential access. 
    *
    * @param    pos
    *           The new read/write position in the stream.
    * 
    * @throws   UnsupportedOperationException 
    *           If the requested operation is not supported.
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public abstract void setPos(long pos)
   throws UnsupportedOperationException,   
          IOException;
   
   /**
    * The length of the stream in bytes.  This does not work for "streams"
    * that require sequential access.
    *
    * @return   The length of the stream.
    * 
    * @throws   UnsupportedOperationException 
    *           If the requested operation is not supported.
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public abstract long getLen()
   throws UnsupportedOperationException,
          IOException;   
   
   /**
    * Truncates or extends the stream to the specified length if this stream
    * supports such an operation. If truncation is requested, all data
    * located after this point in the file is discarded.  If extending the
    * file is requested, the values of the data in the extended portion of
    * the file is undefined. 
    *
    * @param    len
    *           The new length of the file.
    * 
    * @throws   UnsupportedOperationException 
    *           If the requested operation is not supported.
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public abstract void setLen(long len)
   throws UnsupportedOperationException,
          IOException;   
   
   /**
    * Write the given character to the output stream.
    *
    * @param    ch
    *           The character to be written.
    * 
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public abstract void writeCh(char ch)
   throws IOException;

   /**
    * Write the given byte to the output stream.
    *
    * @param    b
    *           The byte to be written.
    * 
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public abstract void writeByte(byte b)
   throws IOException;

   /**
    * Prints a piece of text without newline at the end. Used for PUT statement in batched mode.
    *
    * @param output
    *        The piece of text to be printed. This should be a formatted and pre-arranged piece of text
    *        obtained in <code>Stream.putWorker()</code>.
    */
   public void writeInline(String output)
   {
       BatchHelper.preventGuiBatchStdout();

       ThinClient tc = ThinClient.getInstance();
       tc.flushTerminal();
       tc.setIsNewLineNeeded(true);
       System.out.print(output);
   }

   /**
    * Write the given string to the output stream.
    *
    * @param    data
    *           The data to be written.
    */
   public abstract void write(String data)
   throws IOException;
   
   /**
    * Write the given byte array to the output stream.
    *
    * @param    data
    *           The data to be written.
    *
    * @throws   IOException
    *           If an I/O error occurs.
    */
   public abstract void write(byte[] data)
   throws IOException;
   
   /**
    * Write the specified range of bytes from the given byte array to the output stream.
    *
    * @param    data
    *           The data to be written.
    * @param    off
    *           Starting offset in data from which to read bytes to be written. Must be
    *           non-negative and {@code &lt; data.length}.
    * @param    len
    *           Length of data to be written. Must be non-negative and {@code &lt;= (data.length
    *           - offset)}.
    *
    * @throws   IOException
    *           If an I/O error occurs.
    */
   public abstract void write(byte[] data, int off, int len)
   throws IOException;
   
   /**
    * Peeks at the character from the current read position in the stream (reads a character from
    * the current read position in the stream without incrementing stream read position. The next
    * {@code peekCh()} and {@code readCh()} will return the same value).
    * <p>
    * The underlying stream subclass determines the content of the result. Byte oriented streams
    * such as pipes or files will return a byte while streams that generate keystrokes or
    * characters may return a DBCS or Unicode character.
    *
    * @return  The next character read from the stream, -1 on any failure and -2 upon an
    *          {@code EOF}.
    */
   public abstract int peekCh();
   
   /**
    * Read a character from the current read position in the stream and increment stream pointer.
    * <p>
    * The underlying stream subclass determines the content of the result. 
    * Byte oriented streams such as pipes or files will return a byte while
    * streams that generate keystrokes or characters may return a DBCS or
    * Unicode character.
    *
    * @return   The next character read from the stream, -1 on any failure
    *           and -2 upon an <code>EOF</code>.
    */
   public abstract int readCh();
   
   /**
    * Read all characters from the current read position in the stream to the
    * next line separator (as determined by the <code>File.separator</code>
    * or to the <code>EOF</code>. Any line separator character(s) and the
    * <code>EOF</code> character are not returned.
    *
    * @return   The next line read from the stream.
    * 
    * @throws   IOException
    *           If an I/O error occurs. 
    * @throws   EOFException 
    *           If this input stream reaches the end before reading all the bytes.
    * @throws   InterruptedException
    *           If any thread interrupted the current thread before or while 
    *           the current thread was waiting for a notification. 
    */
   public abstract String readLn()
   throws EOFException,
          IOException,
          InterruptedException;   
   
   /**
    * Closes the input stream and releases OS resources associated with it.
    */
   public abstract void closeIn();
   
   /**
    * Closes the output stream and releases OS resources associated with it.
    */
   public abstract void closeOut();
   
   /**
    * Closes both the input and output streams and releases OS resources
    * associated with it.
    */
   public abstract void close();
   
   /**
    * State of the input side of the stream.
    *
    * @return   <code>true</code> if the input side of the stream is active.
    */
   public abstract boolean isIn();
   
   /**
    * State of the output side of the stream.
    *
    * @return   <code>true</code> if the output side of the stream is active.
    */
   public abstract boolean isOut();
   
   /**
    * Read a single byte from the underlying stream.
    * <p>
    * The default implementation delegates the read to {@link #readCh()}.  TODO: fix this for {@link DirStream}, 
    * {@link ProcessStream} and other sub-classes.
    * 
    * @return   A single byte from the stream.
    */
   public abstract int readByte();

   /**
    * Checks if the stream is alive or not.
    *
    * @return   <code>true</code> if the stream is valid.
    */
   public boolean isValid()
   {
      return isIn() || isOut();
   }
   
   /**
    * State of the suppress form feed flag.
    *
    * @return   copy of the flag
    */
   public boolean isSuppressFormFeed()
   {
      return suppressFormFeed;
   }

   /**
    * Assigns the internal stream reference to the given reference.
    *
    * @param    stream
    *           The new internal stream reference to use for all operations.
    *           
    * @throws   UnsupportedOperationException 
    *           If the requested operation is not supported.
    */
   public abstract void assign(Stream stream)
   throws UnsupportedOperationException;
   
   /**
    * Read a chunk of bytes from the current read position in the stream to the
    * offset calculated by given length or to the <code>EOF</code>, whichever
    * comes first. The <code>EOF</code> character is not returned.
    * 
    * @param    len
    *           The number of bytes to read.
    *
    * @return   The chunk read from the stream or <code>null</code> if there
    *           is nothing to read (the <code>EOF</code> has been reached). The
    *           length of the returned array will be the smaller of the
    *           <code>len</code> parameter or the actual bytes left before the
    *           <code>EOF</code>.
    * 
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public byte[] readBytes(int len)
   throws IOException
   {
      byte[] chunk = new byte[len];
      
      for (int i = 0; i < len; i++)
      {
         int next = readByte();
         
         switch (next)
         {
            case -1:
               // some problem
               String er = String.format("Unknown failure reading at offset %d", i);
               throw new IOException(er);
            case -2:
               if (i == 0)
               {
                  // the first read reached EOF
                  return null;
               }
               else
               {
                  // just return whatever was read
                  return Arrays.copyOf(chunk, i);
               }
            default:
               chunk[i] = (byte) next;
               break;
         }
      }
      
      return chunk;
   }

   /**
    * Read a chunk of characters from the current read position in the stream to the
    * offset calculated by given length or to the <code>EOF</code>, whichever
    * comes first. The <code>EOF</code> character is not returned.
    * 
    * @param    len
    *           The number of characters to read.
    *
    * @return   The chunk read from the stream or <code>null</code> if there
    *           is nothing to read (the <code>EOF</code> has been reached). The
    *           length of the returned array will be the smaller of the
    *           <code>len</code> parameter or the actual bytes left before the
    *           <code>EOF</code>.
    * 
    * @throws   IOException
    *           If an I/O error occurs. 
    */
   public byte[] readChunk(int len)
   throws IOException
   {
      byte[] chunk = new byte[len];
      
      for (int i = 0; i < len; i++)
      {
         int next = readCh();
         
         switch (next)
         {
            case -1:
               // some problem
               String er = String.format("Unknown failure reading at offset %d", i);
               throw new IOException(er);
            case -2:
               if (i == 0)
               {
                  // the first read reached EOF
                  return null;
               }
               else
               {
                  // just return whatever was read
                  return Arrays.copyOf(chunk, i);
               }
            default:
               chunk[i] = (byte) next;
               break;
         }
      }
      
      return chunk;
   }
   
   /**
    * Returns an indication of whether the current page has received any
    * output yet. Page headers are not considered part of the output.
    * <p>
    * The current column has to be zero indicating there is no pending PUT
    * output.
    *
    * @return   <code>true</code> if the page is fresh.
    */
   public boolean isFreshPage()
   {
      int next = rawGetNextLineNum();
      return (next == 1 || next == headerSize + 1) && (column == 0); 
   }
   
   /**
    * Gets the number of available lines (free space) on the current output
    * page.
    *
    * @return   The number of available lines.  This may be positive, 0 or
    *           negative.  It can be negative in the case where the headers
    *           and/or footers for a page are larger than the page size (a
    *           perfectly valid, if silly, thing in 4GL).
    */
   public int getAvailableLines()
   {
      int total = rawGetPageSize();
      int next  = rawGetNextLineNum();
      
      // calculate the amount of used lines
      int taken = next - 1 + (column == 0 ? 0 : 1);
      
      // if we have not output headers yet, account for that
      taken += (next == 1) ? headerSize : 0;
      
      // exclude footers since they will be output at the end
      int rem = total - taken - footerSize;
      return rem;
   }
   
   /**
    * Gets the number of lines on the page excluding footers. Since footers
    * consume part of the lines on a paged stream, this method returns the
    * last effective line that can have non-footer output.
    *
    * @return   The effective page size.
    */
   public int getEffectivePageSize()
   {
      return rawGetPageSize() - footerSize;
   }
   
   /**                         
    * Force new output page. This action writes footers, if any, on the
    * current page and completes the page with the form feed. Next, it
    * renders all page headers on the new page.
    * <p>
    * This method should be called only when the caller guarantees at least
    * one character will be output to the new page. Otherwise the last headers
    * will be there without reason.  
    */
   public void forcePageBreak()
   {
      forcePageBreak(true);
   }
   
   /**                         
    * Force new output page. This action writes footers, if any, on the
    * current page and completes the page with the form feed. Depending on
    * the parameter, it may render all page headers on the new page.
    *
    * @param    headers
    *           <code>true</code> to force the headers on the next page to
    *           be output before returning.
    */
   public void forcePageBreak(boolean headers)
   {
      try
      {
         extendPage();
      }
      catch (IOException ioe)
      {
         ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
      }
      
      writePageBreak();
      
      if (headers)
      {
         processHeaders();
      }
   }
   
   /**
    * Sets contiguous mode for the current page. While <code>true</code> no
    * automatic page break will be generated when output reached the end of
    * the page size.
    * <p>
    * At the end of flushing, the flags will be reset.
    *
    * @param    contiguous
    *           <code>true</code> to enable contiguous mode.
    */
   public void setContiguous(boolean contiguous)
   {
      this.contiguous = contiguous;
   }
   
   /**
    * Gets the contiguous mode state.
    *
    * @return   <code>true</code> is the stream has contiguous mode enabled.
    */
   public boolean isContiguous()
   {
      return contiguous;
   }

   /**
    * Sets the stream font number. The number denotes an index into the font table
    * of the current environment.
    * The method is part of the runtime support of the FONT option of the OUTPUT TO PRINTER
    * (FWD extension to 4GL) statement. This method is only implemented by the printer stream.
    *
    * @param   num
    *          Font number.
    */
   public void setFont(NumberType num)
   {
      if (!num.isUnknown())
      {
         setFont(num.intValue());
      }
   }

   /**
    * Sets the stream font number. The number denotes an index into the font table
    * of the current environment.
    * The method is part of the runtime support of the FONT option of the OUTPUT TO PRINTER
    * statement (FWD extension to 4GL). This method is only implemented by the printer stream.
    *
    * @param   num
    *          Font number.
    */
   public void setFont(int num)
   {
      UnimplementedFeature.unsupported("FONT option is only supported by printer streams.");
   }
   
   /**
    * Detects if need to make a pulse on a empty redirected frame stream.
    *
    * @return   <code>true</code> if need to call pulse on stream closing.
    */
   public boolean getPulseOnClose()
   {
      return pulseOnClose;
   }
   
   /**
    * Sets the need to pulse on close flag.
    *
    * @param   pulseOnClose
    *          New value for pulse on close flag.
    */
   public void setPulseOnClose(boolean pulseOnClose)
   {
      this.pulseOnClose = pulseOnClose;
   }

   /**
    * Detects when a page break is needed.
    *
    * @return   <code>true</code> if a page break should be output.
    */
   boolean needsPageBreak()
   {
      return needsPageBreak(0);
   }
   
   /**
    * Detects when a page break is needed.
    *
    * @param    add
    *           The number of lines of output that are about to be added. If
    *           this value would cause a page to be overflowed, then a page
    *           break will be detected.  May be 0 in the case where this is
    *           called after output is complete.
    *
    * @return   <code>true</code> if a page break should be output.
    */
   boolean needsPageBreak(int add)
   {
      // TODO: should we return false in contiguous mode and/or if the stream is not paged
      
      // NOTE: rawGetNextLineNum represents the number of the next line to be written on, not the
      //       number of filled lines on current page. The last written line is the previous one:
      //       rawGetNextLineNum() - 1
      int sz     = rawGetPageSize();
      int target = rawGetNextLineNum() - 1 + add;
      int bottom = sz - footerSize;
      
      boolean rc = sz > 0 && target > bottom;
      return rc;
   }
   
   /**
    * Gets the current page size in lines.
    *
    * @return   The number of lines per page.
    */
   int rawGetPageSize()
   {
      return pageSize;
   }
   
   /**
    * Sets the current page size in lines.
    *
    * @param    sz
    *           The number of lines per page.
    */
   void rawSetPageSize(int sz)
   {
      if (sz == SET_PAGED_DEFAULT)
         sz = PAGED_DEFAULT;
      
      pageSize = sz;
      pageNum = (sz <= 0) ? 0 : 1;
   }
   
   /**
    * Gets the current page number.
    *
    * @return   The current page number.
    */
   int rawGetPageNum()
   {
      return pageNum;
   }
   
   /**
    * Increments the current page number.
    */
   void rawIncrementPageNum()
   {
      pageNum++;
   }
   
   /**
    * Gets the current page's next line number.
    *
    * @return   The next line number.
    */
   int rawGetNextLineNum()
   {
      return nextLineNum;
   }
   
   /**
    * Increments the current page's next line number.
    */
   void rawIncrementNextLineNum()
   {
      nextLineNum++;
   }
   
   /**
    * Resets the current page's next line number to 1.
    */
   void rawResetNextLineNum()
   {
      nextLineNum = 1;
   }

   /**
    * Check if this is  a file resource.  Implementations will override this as needed.
    * 
    * @return   <code>false</code> as default.
    */
   protected boolean isFileResource()
   {
      return false;
   }
   
   /**
    * Read a block of data from the stream with the block size equal to the size of the binary data array in 
    * the given <code>memptr</code> variable. The data will be written into the memptr starting at the first 
    * byte in the array.  The read data will be the smaller of the block size or the file size.  Only an error 
    * or end of file condition will cause this read to terminate before the block size has been read.
    *
    * @param    var
    *           The variable to which the result is assigned.
    *           
    * @return   The number of the bytes read, -1 on any failure and -2 upon an <code>EOF</code>.
    */
   protected int readBlockRemote(memptr var)
   {
      throw new UnsupportedOperationException();
   }

   /**
    * This method should be called after each write operation. If stream is unbuffered, then
    * call flushData method.
    *
    * @throws   IOException
    *           if an I/O error occurs.
    */
   protected final void afterWrite()
   throws IOException
   {
      if (isUnbuffered())
      {
         flushData();
      }
   }

   /**
    * Flushes this output stream and forces any buffered output bytes to be written out.
    * The general contract of flush is that calling it is an indication that,
    * if any bytes previously written have been buffered by the implementation of the output
    * stream, such bytes should immediately be written to their intended destination.
    *
    * @throws   IOException
    *           if an I/O error occurs.
    */
   protected void flushData() throws IOException
   {
      // no-op
   }

   /**
    * Perform common close <b>preprocessing</b> for all streams (should be
    * called at the <b>beginning</b> of each sub-classes' {@link #close}
    * method.  This allows processing to occur while the stream is still
    * open.
    *
    * @param    in
    *           <code>true</code> if this was triggered during a call to 
    *           {@link #closeIn} or {@link #close}.  Input resources are
    *           to be cleaned up.
    * @param    out
    *           <code>true</code> if this was triggered during a call to 
    *           {@link #closeOut} or {@link #close}.  Output resources are
    *           to be cleaned up.
    */
   protected void cleanup(boolean in, boolean out)
   throws IOException
   {
      if (in)
      {
         doneIn = true;
      }
      
      if (out && !doneOut)
      {
         // this call triggers some final output to the stream. so it has to
         // come before turning on doneOut
         processCloseNotifications();
         
         doneOut = true;

         int pageSz = rawGetPageSize();
         int pageNo = rawGetPageNum();
         
         try
         {
            // paged output?
            if (pageSz > 0)
            {
               int next = rawGetNextLineNum();
               
               // non-empty page?
               // if there is output coming from a put statement on line one,
               // then we still need to output the page-break and footers
               // if there was output with no trailing NL we still need page 
               // break and footer
               if (next > 1                   ||
                   (next == 1 && !trailingNL) ||
                   lastPageWithOutput == pageNo)
               {         
                  if (footers != null)
                  {
                     // fill in a partial page with blanks before processing
                     // any footer or form feed
                     extendPage();
                     buffer = null;
                  }
                  
                  // paged output that has a non-empty page always gets
                  // a form feed output at the end of the file
                  writePageBreak();
               }
            }
         }
         finally
         {
            // flush collected data if any
            endOutput();
          
            // finally inform listeners that stream has been reached end-of-life 
            processUnbindNotifications();
         }
      }
   }
   
   /**
    * Remove the new line character(s) from the given buffer and return the
    * rest of the buffered text as a string.
    *
    * @param    sb
    *           Buffered text, possibly ended by a new line.
    *
    * @return   Any text which exists less the new line character(s).
    */
   protected String readLineCleanup(StringBuilder sb)
   throws EOFException
   {
      int len = sb.length();
      
      if (len == 0)
      {
         throw new EOFException();
      }
      
      int idx = indexOfNewLine(sb);
      
      if (idx < 0)
      {
         idx = len;
      }
      else if (idx > 0)
      {
         char other = sb.charAt(idx - 1);
         
         if (other == '\r')
         {
            // some implementations (even on Linux) may render an extra char
            // even though the platform newline is just a '\n', remove this
            idx--;
         }
      }
      
      // idx == 0 case needs no modification
      
      // return the data less the newline itself
      return sb.substring(0, idx);
   }
   
   /**
    * Helper method to search the position of the new line string in a given
    * text buffer. The method tries to locate system default line separator,
    * if there is no match, Linux style line separator and finally the Windows
    * style line separator. It uses in case the input files are came from OS
    * other than one the P2J runtime is working.
    * <p>
    * WARNING: this looks only in the last two characters in the buffer - if the newline is 
    * expected to be in the middle of the string, an alternative is needed.
    * 
    * @param    sb
    *           Buffered text to find the new line separator string.
    *
    * @return   The position of the new line string or -1 if it is not found.
    */
   protected int indexOfNewLine(StringBuilder sb)
   {
      int l = sb.length();
      if (l == 0)
      {
         return -1;
      }
      
      String s = (l == 1 ? sb.toString() : sb.substring(l - 2, l));
      
      int result = -1;
      for (String nl : NEW_LINES)
      {
         // the lookup must be done using the exact order from NEW_LINES, so a line with '\r\n' is matched
         // with '\r\n' and not '\n'.
         result = s.indexOf(nl);
         if (result >= 0)
         {
            break;
         }
      }
      
      return (result == -1) ? -1 : (l > 1) ? (result + (l - 2)) : result;
   }
   
   /**
    * Check if these two bytes form a new line.
    * 
    * @param    bn_1
    *           First byte.
    * @param    bn
    *           Second byte.
    *           
    * @return   <code>true</code> if the bytes correspond for a new line.
    */
   protected boolean isNewLine(int bn_1, int bn)
   {
      // check the last two read bytes and compare them with the new line
      for (String nl : NEW_LINES)
      {
         if (nl.length() == 1 && nl.charAt(0) == bn)
         {
            return true;
         }
         
         if (nl.length() == 2 && nl.charAt(0) == bn_1 && nl.charAt(1) == bn)
         {
            return true;
         }
      }
      
      return false;
   }
   
   /**
    * Indicate whether or not a lone hyphen on input should be ignored;
    * normal IMPORT statement processing considers this a blank field.
    * Subclasses which require the hyphen to be read as a normal character
    * should override this method.
    * 
    * @return  <code>true</code>.
    */
   protected boolean skipLoneHyphenInput()
   {
      return true;
   }
   
   /**
    * Output a list of fields to the stream. 
    *
    * @param    data
    *           The list of fields to be output.
    * @param    mode
    *           <code>MODE_PUT_NORMAL</code>, <code>MODE_PUT_CONTROL</code>,
    *           <code>MODE_PUT_UNFORMATTED</code> or <code>MODE_EXPORT</code>.
    * @param    delim
    *           The delimiter to use in EXPORT mode.
    *           
    * @throws   ErrorConditionException
    *           On any error during I/O.
    */
   public void putWorker(FieldEntry[] data, int mode, char delim)
   throws ErrorConditionException
   {
      boolean put = (mode != MODE_EXPORT);
      
      try
      {
         switch (mode)
         {
            case MODE_PUT_CONTROL:
               setControl(true);
               break;
            case MODE_PUT_UNFORMATTED:
               setUnformatted(true);
               break;
            case MODE_EXPORT:
               setDelimiter(delim);
               break;
         }
         isPut = put;
         
         if (put)
         {
            // TODO: the nested error mode flag is only used on the server side, we really should only call
            //       this if we are on the server; at this time this causes the TM to be loaded on the client
            //       side which can cause unexpected init of various compents
            
            // enable nested error mode
            TransactionManager.setNestedMode(true);
         }
         
         if (flushBeforePut || !isFileResource())
         {
            // only non-file resources force the flush on each PUT or EXPORT statement.
            flush();
         }

         // we save the put variable so we can use it if we need
         // to restart the output
         savedPut = put;

         // protection from buffer closing by external writer
         synchronized (outActivityLock)
         {
            putWorkerActive = true;
         }
         
         // init our buffer and reset column for EXPORT mode
         startOutput(put ? column : 0);

         // output each field
         for (FieldEntry aData : data)
         {
            // we check for page-break before each field is output
            checkForPageBreak();
            if (aData != null)
            {
               aData.output(this);
            }   
         }
         
         if (!put)
         {
            // now it is safe to close buffer
            synchronized (outActivityLock)
            {
               putWorkerActive = false;
            }
            // flush our buffer and explicitly add NL
            endOutput(true);
         }
         else if (!control)
         {
            // flush output immediately
            try
            {
               String output = currentOut.toString();
                if (this instanceof NullStream &&
                    OutputManager.isBatchInBackground())
               {
                   writeInline(output);
               }
               else
               {
                  write(output);
               }
               currentOut.setLength(0);

               BufferSizeManager.getInstance().notifyOutputOperation(
                                                             true,
                                                             this,
                                                             output.length());
            }
            catch (IOException ioe)
            {
               ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
            }
            finally
            {
               // now it is safe to close buffer
               synchronized (outActivityLock)
               {
                  putWorkerActive = false;
               }
            }
         }
         else
         {
            // now it is safe to close buffer
            synchronized (outActivityLock)
            {
               putWorkerActive = false;
            }
         }
      }
      catch (StopConditionException err)
      {
         // pipe has been broken
         throw err;
      }
      catch (ConditionException err)
      {
         if (put)
         {
            // allow caller to see the error (silent mode will never be
            // active so this will always throw an error)
            ErrorManager.recordOrThrowError(564,
                                            "Unable to evaluate expression for PUT statement");
         }
         else
         {
            // allow caller to see the original error directly in export mode
            throw err;
         }
      }
      
      finally
      {
         if (put)
         {
            // TODO: only call this on the server
            
            // disable nested error mode
            TransactionManager.setNestedMode(false);
         }
         
         switch (mode)
         {
            case MODE_PUT_CONTROL:
               setControl(false);
               break;
            case MODE_PUT_UNFORMATTED:
               setUnformatted(false);
               break;
            case MODE_EXPORT:
               setDelimiter(null);
               break;
         }
         isPut = false;
         flushBeforePut = false;
      }
   }
   
   /**
    * Check if current stream is bound to terminal.
    * 
    * @return   <code>true</code> if stream is terminal.
    */
   public boolean isTerm()
   {
      return false;
   }

   /**
    * Number of lines pending for output but not buffered in local stream
    * buffer.
    * 
    * @return  Number of pending lines.
    */
   public int getPendingRowCount()
   {
      return pendingRowCount;
   }

   /**
    * Notify stream about pending rows.
    * 
    * @param    rows
    *           Number of pending rows.
    */
   public void setPendingRowCount(int rows)
   {
      pendingRowCount = rows;
   }
   
   /**
    * Read a block of data from the stream with the block size equal to the size of the binary data array in 
    * the given <code>memptr</code> variable. The data will be written into the memptr starting at the first 
    * byte in the array.  The read data will be the smaller of the block size or the file size.  Only an error 
    * or end of file condition will cause this read to terminate before the block size has been read.
    * <p>
    * This API is used by the FWD client-side.
    * 
    * @param    var
    *           The variable to which the result is assigned.
    *           
    * @return   The number of the bytes read, -1 on any failure and -2 upon an <code>EOF</code>.
    */
   int readBlockWorker(memptr var)
   {
      // TODO: memptr may be on server-side, improve this
      
      long len = var.lengthOf();
      byte[] bytes = readBytesWorker(len);
      
      if (bytes == null)
      {
         return -1; // unexpected error
      }
      
      if (bytes.length == 0)
      {
         return -2; // EOF
      }
      else
      {
         var.setBytes(new raw(bytes), 1);
      }
      
      return bytes.length;
   }
   
   /**
    * Read a chunk of bytes from the specified stream, of at most the given length.
    * 
    * @param   len
    *          The maximum length of bytes to read.
    *
    * @return  {@code null} in case of errors or the read bytes otherwise.
    */
   private byte[] readBytesWorker(long len)
   {
      byte[] res = new byte[(int) len];
      int readLen = -1;
      
      for (int i = 0; i < len; i++)
      {
         int ch = readByte();
         if (ch == -1)
         {
            return null;
         }
         
         if (ch == -2)
         {
            readLen = i;
            break;
         }
         
         res[i] = (byte) ch;
      }
      
      if (readLen != -1 && readLen != len)
      {
         byte[] res2 = new byte[readLen];
         System.arraycopy(res, 0, res2, 0, readLen);
         res = res2;
      }
      
      return res;
   }

   /**
    * Extend the output of a partial page (with blank lines) to the end of 
    * valid output so that any footer and a form feed can be output to
    * terminate the page at the proper minimum length.  This page extend
    * feature only occurs if footers are being used.
    */
   private void extendPage()
   throws IOException
   {
      if (footers != null)
      {
         int len = (rawGetPageSize() - footerSize);
         int from = rawGetNextLineNum();
         
         for (int i = from;  i <= len; i++)
         {
            write(NEWLINE);
            rawIncrementNextLineNum();
         }
      }
   }
   
   /**
    * Obtain the next line of input and translate any errors or exceptional
    * conditions into their Progress equivalents.
    *
    * @param    inString
    *           <code>true</code> if the special EOF processing (when the '.'
    *           appears by itself on its own line) should NOT be honored.
    *           This only occurs during formatted reading when a string is
    *           being processed.
    *
    * @return   The next line of input or <code>null</code> in the case of
    *           and error condition raised during silent error mode.
    *
    * @throws   EndConditionException
    *           Upon reading at EOF (not up to EOF but the read that occurs
    *           where EOF is the next character to be read).
    * @throws   ErrorConditionException
    *           If an IOException occurs.
    * @throws   StopConditionException
    *           If the read was interrupted.
    */
   private String reload(boolean inString)
   throws EndConditionException,
          ErrorConditionException,
          StopConditionException
   {
      String next = null;
      
      try
      {
         next = readNextLine(inString, false);
      }
      catch (EOFException eof)
      {
         throw new EndConditionException("EOF");
      }
      catch (InterruptedIOException | InterruptedException iioe)
      {
         ErrorManager.handleStopException(new StopConditionException(iioe));
      }
      catch (IOException ioe)
      {
         // TODO: actual Progress compatible text and number are unknown
         ErrorManager.recordOrThrowError(-1, "IOException during read.");
         return null;
      }
      
      return next;
   }
   
   /**
    * Read the next line from the stream using Progress rules. This must 
    * properly handle the <code>EOF</code> event (as reported by the stream),
    * the <code>EOF</code> character (0x1A or CTRL-Z), the fake Progress 
    * <code>EOF</code> character (a "." by itself on its own line) and the
    * line continuation characters of tilde and backslash.
    * <p>
    * If the <code>null</code> character is detected in the read data, all 
    * data from that character through the next <code>newline</code>
    * (inclusive) will be discarded.  This is effectively, another form of
    * line continuation as the next line will be seamlessly concatenated to
    * this result. The only exception to this is when there is no following 
    * <code>newline</code> on the NEXT line in which case ALL data read is 
    * discarded and the empty string is returned (WARNING: the EOF event is
    * not raised until the next call to this method in this case).
    *
    * @param    inString
    *           <code>true</code> if the special EOF processing (when the '.'
    *           appears by itself on its own line) should NOT be honored.
    *           This only occurs during formatted reading when a string is
    *           being processed.
    *           
    * @param    isImportUnformatted
    *           <code>true</code> if the import has the unformatted option.
    *
    * @return   The next line from the stream. 
    */
   private String readNextLine(boolean inString, boolean isImportUnformatted)
   throws EOFException,
          IOException,
          InterruptedException
   {
      // reset flag
      nullQuirk = false;
         
      StringBuilder sb = new StringBuilder();
      
      String next = readLineDropNull();
      
      if (next == null)
      {
         throw new EOFException("");
      }
      
      // process all line continuations
      while (next.endsWith("~") || next.endsWith("\\"))
      {
         sb.append(next, 0, next.length() - 1);
         next = readLineDropNull();
      }
      
      // add the only segment or the last segment
      sb.append(next);
            
      String res = sb.toString();
      
      if ((res.equals(".") && !inString && !isImportUnformatted) ||
          (res.length() == 1 && res.charAt(0) == EOF))
      {
         throw new EOFException("");
      }
      
      return res;
   }
   
   /**
    * Read the next line from the stream dropping all data at and after any
    * <code>null</code> character including the next <code>newline</code>.
    * This is effectively, another form of line continuation as the next line
    * will be seamlessly concatenated to this result.
    * <p>
    * The only exception to this is when there is no following
    * <code>newline</code> on the NEXT line in which case ALL data read is 
    * discarded and the empty string is returned (WARNING: the EOF event is
    * not raised until the next call to this method in this case). Note that
    * in this case an internal flag is set which can be checked in the
    * caller. 
    *          
    * @return   The next line from the stream. 
    */
   private String readLineDropNull()
   throws EOFException,
          IOException,
          InterruptedException
   {  
      // this may be null or throw EOFException in the case of EOF (the
      // behavior depends on the backing stream implementation)
      String next = readLn();
      
      if (next != null)
      {
         int idx = next.indexOf('\0');
         
         if (idx != -1)
         {
            // embedded null case so truncate the current line
            StringBuilder sb = new StringBuilder(next.substring(0, idx));
   
            try
            {
               // continue data with the next line
               next = readLineDropNull();
            }
            catch (EOFException eof)
            {
               // eat this because we must be able to silently return in the
               // case where there is no line delimiter on the NEXT line after
               // an embedded null case occurs
               next = null;
            }
            
            if (next == null)
            {
               nullQuirk = true;
            }
            else
            {
               sb.append(next);
            }
            
            next = sb.toString();
         }
      }
      
      return next;
   }
   
   /**
    * Add a line end and if in paged output mode and the current page is
    * full, add a page break (the form feed character 0x0C) to the current
    * write buffer and maintain the next line number and current page number.
    * <p>
    * If in control mode, no page breaks will be emitted and the page/line
    * counters will not be modified.
    * <p>
    * If <code>conditional</code> parameter is set to <code>true</code> then
    * new line is appended only if buffer does contain new line at the end. 
    * 
    * @param    conditional
    *           If this parameter is set to <code>true</code> then appending 
    *           of new line is conditional.
    * @param    noPageBreak
    *           If this parameter is set to <code>true</code> then there is no
    *           checking for page break. Instead, the line-counter is 
    *           incremented.          
    */
   private void processLineEnd(boolean conditional, boolean noPageBreak)
   throws ErrorConditionException
   {
      if (!conditional || !trailingNL)
      {
         writeNewLine();

         if (noPageBreak)
         {
            rawIncrementNextLineNum();
         }
         else
         {
            processPageBreak();
         }
      }
   }
   
   /**
    * Add the platform-specific new line to the current write buffer.
    */
   private void writeNewLine()
   throws ErrorConditionException
   {
      append(NEWLINE);
      
      if (!control)
      {
         column = 0;
      }
   }
   
   /**
    * Set the NEWLINE state.
    * 
    * @param    trailingNL
    *           Track trailing NEWLINE state.
    */
   private void setTrailingNL(boolean trailingNL)
   {
      this.trailingNL = trailingNL;
      this.trailingWithoutPutNL = trailingNL & (!isPut);
   }

   /**
    * If in paged output mode and the current page is full, add a page break
    * (the form feed character 0x0C) to the current write buffer and maintain
    * the next line number and current page number.
    * <p>
    * If in control mode, no page breaks will be emitted and the page/line
    * counters will not be modified (this is a NOP).
    */
   private void processPageBreak()
   {
      if (!control)
      {
         // determine if we are in paged mode AND if we should be starting the next page
         if (needsPageBreak())
         {
            // move to a new page
            writePageBreak();
         }
         else
         {
            // just increment our counter
            rawIncrementNextLineNum();
         }
      }      
   }
   
   /**
    * Add any needed footer(s) and then a page break (the form feed character
    * 0x0C) to the current stream (buffered or not buffered), then increment
    * the current page number and reset the next line number to 1.
    */
   private void writePageBreak()
   throws ErrorConditionException
   {
      if (footersEmitted)
      {
         // if the footers were output to the stream, this means the 
         // page-break was also written - so we do nothing more.
         return;
      }
      
      // output footer
      if (footers != null)
      {
         processFooters();
      }
      
      try
      {
         // output form feed
         if (!suppressFormFeed)
         {
            try
            {
               if (currentOut == null)
               {
                  try
                  {
                     // unbuffered mode
                     write("\f");
                  }
                  catch (IOException ioe)
                  {
                     ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
                     return;
                  }
               }
               else
               {
                  // buffered output
                  append("\f");
               }
            }
            finally
            {
               // maintain paging state
               rawIncrementPageNum();
               rawResetNextLineNum();
               setTrailingNL(true);
            }
         }
      }
      finally
      {
         // send notifications
         processPageBreakNotifications();
         
         suppressFormFeed = false;
      }
   }   
   
   /**
    * Notify all registered listeners that a page break has been written for
    * this stream.
    * Notified listeners are automatically removed from the registry.
    */
   private void processPageBreakNotifications()
   {
      if (pageListeners != null)
      {
         // iterate on the copy so that the notification handler can re-register if needed
         Map<PageBreakListener, Object> copy = new HashMap<>(pageListeners);
         Iterator<PageBreakListener> list = copy.keySet().iterator();
         
         while (list.hasNext())
         {
            PageBreakListener pl = list.next();
            pl.notifyPageBreak(this);
            pageListeners.remove(pl);
         }      
      }
   }
   
   /**
    * Notify all registered listeners that this stream is being closed.
    */
   private void processCloseNotifications()
   {
      if (closeListeners == null)
         return;
      
      for (CloseListener cl : closeListeners.keySet())
      {
         cl.notifyClose(this);
      }
   }
   
   /**
    * Notify all registered listeners that this stream is dead.
    */
   private void processUnbindNotifications()
   {
      if (closeListeners == null)
         return;

      for (CloseListener cl : closeListeners.keySet())
      {
         cl.notifyUnbind(this);
      }

      closeListeners.clear();
      closeListeners = null;
   }
   
   /**
    * Notify all registered listeners that this stream is advancing page.
    */
   private void processAdvanceNotifications()
   {
      if (advanceListeners != null)
      {
         Iterator<AdvanceListener> list = advanceListeners.iterator();
         
         while (list.hasNext())
         {
            AdvanceListener al = list.next();
            al.notifyAdvance(this);
            list.remove();
         }      
      }
   }
   
   /**
    * Iterate through the given set of {@link RepeatedOutput} instances and
    * for each element, invoke the {@link RepeatedOutput#render} method. 
    *
    * @param    out
    *           The list of instances to use for rendering.
    * @param    footers
    *           When set to <code>true</code>, it means we output the page 
    *           footers; else, we output the headers.
    */
   private void processRendering(Set<RepeatedOutput> out, boolean footers)
   {
      // don't let the processing of headers OR of footers trigger recursive
      // emission of footers
      footersEmitted = true;
      
      if (out == null)
         return;


      // casting works only if streams are not using rpc
      StreamDaemon streamHelper = (StreamDaemon) OSResourceManager.getStreamHelper();
      
      // make sure the output is switched to this stream temporarily
      int sid = streamHelper.idFromStream(this);
      int oid = -2;

      ThinClient tc = ThinClient.getInstance();
      if (!tc.isRedirected(this))
      {
         oid = tc.switchOutput(sid, false);
      }

      Iterator<RepeatedOutput> list = out.iterator();
      RedirectedTerminal rt = ThinClient.redirectedTerminalForStream(this);
      
      while (list.hasNext())
      {
         RepeatedOutput ro = list.next();
         ro.render(this);

         // flush after every page element
         rt.flush();

         try
         {
            flushControlled(true);
         }
         catch (IOException ioe)
         {
         }

         // if we output the headers, then the column field must be set to zero - this is because
         // the put statement must output on a new line, not near the page-break character
         if (!footers)
         {
            column = 0;
         }
      }

      if (oid != -2)
      {
         tc.switchOutput(oid, false);
      }
      
      footersEmitted = false;
   }
   
   /**
    * Append the given string to the output buffer and maintain the column
    * counter. If in <code>control</code> mode, the output buffer is
    * bypassed (the output is written directly to the stream) and no updates
    * are made to the line/page counters.
    *
    * @param    data
    *           The string to append.
    */
   private void append(String data)
   throws ErrorConditionException,
          StopConditionException
   {
      setTrailingNL(data.endsWith(NEWLINE));
      
      if (!control)
      {
         column += data.length();
         currentOut.append(data);
      }
      else
      {
         try
         {
            write(data);
         }
         
         catch (InterruptedIOException iioe)
         {
            ErrorManager.handleStopException(new StopConditionException(iioe));
         }
         
         catch (IOException ioe)
         {
            ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
         }
      }
   }
   
   /**
    * Append the given <code>char</code> to the output buffer and maintain
    * the column counter.  If in <code>control</code> mode, the output buffer is
    * bypassed (the output is written directly to the stream) and no updates
    * are made to the line/page counters.
    *
    * @param    data
    *           The <code>char</code> to append.
    */
   private void append(char data)
   throws ErrorConditionException,
          StopConditionException
   {
      setTrailingNL(NEWLINE_LEN == 1 && NEWLINE.charAt(0) == data);
      
      if (!control)
      {
         column += 1;
         currentOut.append(data);
      }
      else
      {
         try
         {
            writeCh(data);
         }
         catch (InterruptedIOException iioe)
         {
            ErrorManager.handleStopException(new StopConditionException(iioe));
         }
         catch (IOException ioe)
         {
            ErrorManager.showErrorAndAbend(140, PIPE_BROKEN_MESSAGE);
         }
      }
   }
   
   /**
    * Generates formatted data in conformity with PUT / PUT UNFORMATTED / PUT
    * CONTROL specifications.
    * <p>
    * Formatting rules are specified in the table below, but please read
    * comments to the table first:
    * <ul>
    *    <li>
    *      By the words "specific format" we mean one of the following
    *      (in preference order):
    *     <ol>
    *       <li> 
	 *          explicit format string that was specified into PUT or
    *          PUT UNFORMATTED statement;
    *       </li>
    *       <li>
	 *          explicit format string that was specified into variable
    *          definition statement;
    *       </li>
    *       <li> default type-specific format. </li>
    *     </ol>
    *    </li>
    *    <li>
    *       By the words "left-aligned up to some length" we mean that
    *       trailing spaces were added to the formatted value in order to
    *       match given length.
    *    </li>
    *    <li>
    *       By the words "right-aligned up to some length" we mean that
    *       leading spaces were added to the formatted value in order to match
    *       given length.
    *    </li>
    *    <li>
    *       By the words "string format" we mean string representation of the
    *       instance data in a form that is compatible with the MESSAGE
    *       language statement.
    *    </li>
    * </ul>
    * <p>
    * <table style="text-align: left; width: 100%;" border="1" summary="">
    * <tr>
    *    <td></td>
    *    <td style="font-weight: bold;">
    *       put (with or without format string) / put unformatted with format
    *       string specification
    *    </td>
    *    <td style="font-weight: bold;">
    *       put unformatted without format string specification / put control
    *    </td>
    * </tr>
    * <tr>
    *    <td> integer, not unknown </td>
    *    <td> specific format, right-aligned up to format length </td>
    *    <td> string format </td>
    * </tr>
    * <tr>
    *    <td> decimal, not unknown </td>
    *    <td> specific format, right-aligned up to format length </td>
    *    <td> string format </td>
    * </tr>
    * <tr>
    *    <td> character, not unknown </td>
    *    <td> specific format</td>
    *    <td> string format </td>
    * </tr>
    * <tr>
    *    <td> date, not unknown </td>
    *    <td> specific format</td>
    *    <td> string format </td>
    * </tr>
    * <tr>
    *    <td> logical, not unknown </td>
    *    <td>
    *       specific format, left-aligned up to maximum between positive
    *       value length and negative value length
    *    </td>
    *    <td> string format </td>
    * </tr>
    * <tr>
    *    <td> recid, not unknown </td>
    *    <td> specific format, right-aligned up to format length </td>
    *    <td> string format </td>
    * </tr>
    * <tr>
    *    <td> unknown integer </td>
    *    <td> ?, right-aligned up to specific format length </td>
    *    <td> ? </td>
    * </tr>
    * <tr>
    *    <td> unknown decimal </td>
    *    <td> ?, right-aligned up to specific format length </td>
    *    <td> ? </td>
    * </tr>
    * <tr>
    *    <td> unknown character </td>
    *    <td> ?, left-aligned up to specific format length </td>
    *    <td> ? </td>
    * </tr>
    * <tr>
    *    <td> unknown date </td>
    *    <td>
    *       string consisting of spaces which number equals specific format
    *       length
    *    </td>
    *    <td> ? </td>
    * </tr>
    * <tr>
    *    <td> unknown logical </td>
    *    <td>
    *       ?, left-aligned up to maximum between positive value length and
    *       negative value length in the specific format
    *    </td>
    *    <td> ? </td>
    * </tr>
    * <tr>
    *    <td> unknown recid </td>
    *    <td> ?, right-aligned up to specific format length </td>
    *    <td> ? </td>
    * </tr>
    * <tr>
    *    <td> unknown literal </td>
    *    <td>
    *       if explicit format string is specified, then it is ?, left-aligned
    *       up to explicit format length, otherwise it is ? followed by three
    *       spaces
    *    </td>
    *    <td> ? </td>
    * </tr>
    * </table>
    *
    * @param   val
    *          The data to be output.
    * @param   fmt
    *          The format string to use. If <code>null</code>, the default
    *          type-specific format will be used.
    *
    * @return  Formatted data.
    */
   private String generateFormattedText(BaseDataType val, String fmt)
   {
      String text = null;
      
      if ((val instanceof logical) && (fmt == null && !unformatted))
      {
         fmt = val.defaultFormatString();
      }
      
      if ((!unformatted || fmt != null) && !control)
      {
         String defFormat = null;
         if (val instanceof character)
         {
            defFormat = generateStringFormat(null, val.toStringMessage().length());
         }
         else
         {
            defFormat = val.defaultFormatString();
         }
         
         boolean isNumberType =  val instanceof NumberType;
         
         boolean isDefaultFormat = fmt != null && fmt.equals(defFormat);
         
         if (isPut && isDefaultFormat && !(isNumberType))
         {
            //do not use default format, only explicit format is allowed in unformatted mode
            text = val.toStringMessage();
         }
         else
         {
            text = val.toString(fmt);
         }
      }
      else if (val instanceof raw && !val.isUnknown())
      {
         text = new String(((raw) val).getByteArray());
      }
      else
      {
         text = val.toStringMessage();
      }
      
      if ((!unformatted || fmt != null) && !control)
      {
         if (val instanceof NumberType)
         {
            int len = (fmt == null) ? val.defaultFormatString().length()
                                    : NumberType.getValueTextLength(text, fmt);
            
            text = StringHelper.rightAlignText(text, len);
         }
         else if (((val instanceof logical) || (val instanceof character)) && val.isUnknown())
         {
            int len = val.formatLength(fmt);
            text = StringHelper.leftAlignText(text, len);
         }
         else if ((val instanceof date) && val.isUnknown())
         {
            int len = (fmt == null) ? val.defaultFormatString().length() : fmt.length();
            text = StringHelper.repeatChar(' ', len);
         }
         else if (val instanceof unknown)
         {
            int len = (fmt == null) ? 4 : val.formatLength(fmt);
            text = StringHelper.leftAlignText("?", len);
         }
      }
      
      return text;
   }
   
   /**
    * This method is a custom algorithm for page-break checking - the skip 
    * option with a put stream ... statement will emit the page-break only if
    * <code>LINE-COUNTER &ge; PAGE-SIZE + 1</code>  
    */
   private void checkForPageBreak() 
   {
      int pageSz = rawGetPageSize();
      int lineCounter = rawGetNextLineNum();
      int bottom = pageSz - footerSize;
      
      // a page break will be emitted only if we are beyond line PAGE-SIZE + 1
      if (pageSz > 0 && lineCounter >= bottom + 1) 
      {
         writePageBreak();
         
         // we need to check if page headers are needed 
         processHeaders();
         
         // re-start the output, if the footers or headers end it
         if (currentOut == null)
         {
            startOutput(savedPut ? column : 0);
         }
      }
   }
}