FileSystemDaemon.java

/*
** Module   : FileSystemDaemon.java
** Abstract : worker object that handles delegated file system processing
**
** Copyright (c) 2006-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
** 001 GES 20060211   @24539 Created initial version, which completely
**                           implements the FileSystem interface.
** 002 GES 20070914   @35141 Refactored the propath searching code to
**                           use a static worker. This allows the same
**                           routine to be used at conversion time for
**                           searching.
** 003 GES 20080118   @36862 Added NPE protection logic in search().
** 004 GES 20080123   @36903 More NPE protection logic.
** 005 GES 20080321   @37630 Added more data (almost complete) to the
**                           FILE-INFO support.
** 006 GES 20110712          Searching now honors the case-sensitivity of the
**                           legacy system.
** 007 GES 20111025          Modified API export code to capture and use the
**                           authentication token so that multiple registrations
**                           can be done per JVM session. This occurs when the
**                           client code is running in the server process and
**                           there is a CTRL-C that causes the client to
**                           restart.
** 008 GES 20111111          Modified interface export to allow registration
**                           to be exclusively local or remote as needed.
** 009 CS  20121104          Implement PROPATH assignment.
** 010 CS  20130115          Task 1613: finish FILE-INFO system handle support
** 011 CA  20140313          Fixed the regexp for win-separator replacement.
**                           Added delete(File, recursive). 
** 012 ECF 20150715          Replace StringBuffer with StringBuilder.
** 013 EVL 20151026          Adding more criteria to detect file path relative to the current
**                           directory.  For UNIX-like systems added the path beginning not with
**                           File separator.  For Windows systems  the current path must not start
**                           with File separator and root drive letter + ':\\'.  The canonical
**                           name is now used for this case too in getCaseInsensitiveMatch()
**                           filename transformation.  If root or current path search is failed
**                           the PROPATH entries are used as last resort.
** 014 IAS 20160113          Added image files' search with different extensions
** 015 EVL 20160329          Adding the ability to check the environment variable from OS first
**                           and if not found - use defined in JVM or via -D java start option.
** 016 OM  20160428          Added searchAll() method.
** 017 SVL 20171109          Added getFileContents.
** 018 HC  20171228          Fixed normalization of file path separators.
** 019 ECF 20180109          Normalize file separators for file copy.
**     GES 20180115          Added normalization to all other filename inputs so that all forms
**                           of usage are safe.
** 020 EVL 20180205          Fix for Windows Matcher.replaceAll() method issue with file
**                           separator as input parameter.
** 021 SBI 20180518          Added PathResolver usages in order to resolve a legacy file system
**                           path on the host system.
**     CA  20180520          Fixed getFileName - if the file/folder exists, use it, regardless of
**                           case-sensitivity.
** 022 EVL 20180530          Adding support for (GET|SET)-WORKING-DIRECTORY function.
**     ECF 20180609          Removed static modifier from {get|set}WorkingDirectory methods.
**                           Reorganized methods to meet coding standards.
** 023 SBI 20180921          Changed mkdir to do a case-insensitive search of existing parents.
** 024 AIL 20200211          Empty path for mkdir should return no-error.
** 025 AIL 20210813          Added getSearchPath() method.
**     CA  20220220          Fixed copy - if the target is a directory, append the source filename to it.
** 026 HC  20230427          Implemented server-side file-system resource. File operations
**                           performed on client can be optionally performed directly on server
**                           without the expensive remote call.
** 027 CA  20230510          'fsd' may be null if 'getFileName' is called from conversion.
** 028 RFB 20231018          Added capability to handle a path starting with ~/ in normalizeFileSeparators.
**                           Ref. #7858.
** 029 GBB 20240826          Renaming 'single' to 'local' (single client removed).
**                           Moving getTempDirectory() here from OsPropertiesDaemon.
** 030 GBB 20240912          Moving file / dir stream methods to FileSystem.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.util.osresource;

import java.io.*;
import java.nio.file.*;
import java.util.*;
import java.util.regex.*;

import com.goldencode.p2j.net.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.security.SecurityManager;
import com.goldencode.p2j.util.*;
import com.goldencode.util.*;

/**
 * Implements file-system utility methods (for query and manipulation) in a
 * manner that is suitable for use on a remote node.  Construction of this
 * object causes these APIs to be exported from the current node.
 */
public class FileSystemDaemon
implements FileSystem
{
   /** Regexp pattern for matching any path separators */
   private static final Pattern anyPathSep = Pattern.compile(ANY_PATH_SEPARATOR);
   
   /**
    * Maximum size of file that can be transferred using {@link #getFileContents(String)}, in
    * bytes.
    */
   private static final int MAXIMUM_FILE_TRANSFER_SIZE = 5 * 1024 * 1024;
   
   /** Flag to note if the rutnime platform for this JVM is Windows. */
   private static final boolean isWin = PlatformHelper.isUnderWindowsFamily();
   
   /** The singleton instance of this class */
   private static final ContextLocal<FileSystemDaemon> instance = new ContextLocal<>();
   
   /** Token used to authenticate with the dispatcher when registering APIs. */
   private Object modToken = null;

   /** Cached server-side flag */
   private final boolean isServerSideFS;

   /** Paths to search for relative filenames. */
   private String[] paths = null;
   
   /** The legacy system's filesystem case-sensitivity. */
   private boolean caseSens = true;
   
   /** The legacy path resolver */
   private PathResolverContainer pathResolver;

   /** File system resource plugin */
   private FileSystemResource fsRes;

   /**
    * Create an instance and export its API to the network.
    *
    * @param    local
    *           <code>true</code> to register purely on a local basis (no remote/network access to these 
    *           methods).
    * @param    serverSideFS
    *           <code>true</code> if the server-side file system is being used.
    */
   private FileSystemDaemon(boolean local, boolean serverSideFS)
   {
      synchronized (FileSystemDaemon.class)
      {
         modToken = RemoteObject.registerServer(FileSystem.class, this, modToken, local);
         SecurityManager sm = SecurityManager.getInstance();
         fsRes = serverSideFS ?
            (FileSystemResource) sm.getMandatoryPluginInstance(FileSystemResource.TYPE_NAME) :
            (FileSystemResource) sm.getPluginInstance(FileSystemResource.TYPE_NAME);
         isServerSideFS = serverSideFS;
      }
   }
   
   /**
    * Retrieve the target instance.
    *
    * @param    local
    *           <code>true</code> to register purely on a local basis (no remote/network access to these 
    *           methods).
    * @param    isServerSideFS
    *           <code>true</code> if the server-side file system is being used.
    */
   static synchronized FileSystemDaemon getInstance(boolean local, boolean isServerSideFS)
   {
      FileSystemDaemon fsd = instance.get();
      
      if (fsd == null)
      {
         fsd = new FileSystemDaemon(local, isServerSideFS);
         instance.set(fsd);
      }
      
      return fsd;
   }
   
   /**
    * Search for the given file in each directory listed in the paths specified. If the file
    * system object exists and is a file, then the filename is returned. If the path list contains
    * an empty string or a '.', then the current directory is searched, otherwise only the listed
    * directories are searched.
    * <p>
    * If the filename is specified as an absolute name, this specific filename is checked and no
    * path searching is done.
    *
    * @param   paths
    *          The list of paths to search. The file separator character used in these paths
    *          should be the same as the current system's file separator character.
    * @param   caseSens
    *          {@code true} if the search should honor a case-sensitive file-system.  When set to
    *          {@code false} and the current filesystem is naturally case-sensitive, a
    *          case-insensitive search will be implemented.
    * @param   filename
    *          The absolute or relative filename for which to search.
    * @param   allowDir
    *          If {@code true} the search will also directories, if {@code false} only file
    *          results will be returned.
    *
    * @return  If the file is found in the current directory, the relative filename is returned.
    *          If the file is absolute or found in any other path, the full absolute filename is
    *          returned.  If the file is not found or the referenced file system object is not a
    *          file, {@code null} is returned.
    */
   public static String search(String[] paths,
                               boolean  caseSens,
                               String   filename,
                               boolean  allowDir)
   {
      return search(paths, caseSens, filename, allowDir, false);
   }

   /**
    * Search for the target path name. If the target path name is an absolute name or starts from
    * the dot path name, then this specific filename is checked and no path searching is done if
    * the special searchAbsoluteOrRelativeOnly is set on. Otherwise the given path list is used
    * to resolve the target path name against them and the first matched file is returned.
    *
    * @param   paths
    *          The list of paths to search. The file separator character used in these paths
    *          should be the same as the current system's file separator character.
    * @param   caseSens
    *          {@code true} if the search should honor a case-sensitive file-system.  When set to
    *          {@code false} and the current filesystem is naturally case-sensitive, a
    *          case-insensitive search will be implemented.
    * @param   filename
    *          The absolute or relative filename for which to search.
    * @param   allowDir
    *          If {@code true} the search will also directories, if {@code false} only file
    *          results will be returned.
    * @param   searchAbsoluteOrRelativeOnly
    *          The true value indicates that the given absolute or relative path name is only
    *          looked up, otherwise the given list of paths are used to look up the target path.
    *
    * @return  If the file is found in the current directory, the relative filename is returned.
    *          If the file is absolute or found in any other path, the full absolute filename is
    *          returned.  If the file is not found or the referenced file system object is not a
    *          file, {@code null} is returned.
    */
   public static String search(String[] paths,
                               boolean  caseSens,
                               String   filename,
                               boolean  allowDir,
                               boolean  searchAbsoluteOrRelativeOnly)
   {
      // make sure there is something to search for (the code below needs to be protected)
      if (filename == null)
      {
         return null;
      }
      
      // if the the OS is windows replace all / to \ from the search paths
      String fn = normalizeFileSeparators(filename);
      
      String result = getAbsoluteOrCurrent(caseSens, fn, allowDir);
      
      if (result != null || searchAbsoluteOrRelativeOnly)
      {
         return result;
      }
      
      // not found in root or current, try PROPATH entries
      if (paths == null)
      {
         return null;
      }
      
      return Utils.tryWhileNull(Arrays.asList(paths), 
                                path -> getInPath(caseSens, path, fn, allowDir));
   }
   
   /**
    * Return all files matching the specified {@code filename} from each directory listed in the
    * {@code paths} specified. If the path list contains an empty string or a '.', then the
    * current directory is searched, otherwise only the listed directories are searched.
    * <p>
    * If the filename is specified as an absolute name, this specific filename is checked and no
    * path searching is done and returned as a single element array.
    *
    * @param   paths
    *          The list of paths to search. The file separator character used in these paths
    *          should be the same as the current system's file separator character.
    * @param   caseSens
    *          {@code true} if the search should honor a case-sensitive file-system.  When set to
    *          {@code false} and the current filesystem is naturally case-sensitive, a
    *          case-insensitive search will be implemented.
    * @param   filename
    *          The absolute or relative filename for which to search.
    * @param   allowDir
    *          If {@code true} the search will also directories, if {@code false} only file(s)
    *          results will be returned.
    *          
    * @return  The relative filename for all matching files are returned as {@code String} list.
    *          If the file is absolute or found in any other path, the full absolute filename is
    *          returned as a single element. If the file is not found or the referenced file
    *          system object is not a file, {@code null} is returned.
    */
   public static List<String> searchAll(String[] paths,
                                        boolean  caseSens,
                                        String   filename,
                                        boolean  allowDir)
   {
      // make sure there is something to search for (the code below needs to be protected)
      if (filename == null)
      {
         return null;
      }
      
      // if the the OS is windows replace all / to \ from the search paths
      String fn = normalizeFileSeparators(filename);
      
      String result = getAbsoluteOrCurrent(caseSens, fn, allowDir);
      
      if (result != null)
      {
         List<String> ret = new ArrayList<>(1);
         ret.add(result);
         return ret;
      }
      
      // not found in root or current, try PROPATH entries
      if (paths == null)
      {
         return null;
      }
      
      ArrayList<String> ret = new ArrayList<>(paths.length);
      
      for (String path : paths)
      {
         String fileName = getInPath(caseSens, path, fn, allowDir);
         
         if (fileName != null)
         {
            ret.add(fileName);
         }
      }
      
      ret.trimToSize();
      return ret.isEmpty() ? null : ret;
   }
   
   /**
    * Gets the OS environment variable value for given name if devined.
    *
    * @return   The currently defines ins OS environment value or <code>null</code> if variable 
    *           is currentlu not defined for given process.
    */
   public static native String getEnvironmentValue(String vname);
   
   /**
    * Gets the working directory from OS.
    *
    * @return   The current working directory.
    */
   public static native String getWorkingDir();

   /**
    * Sets the working directory via native OS call.
    *
    * @param    newWorkingDir
    *           A new working directory to set.
    *           
    * @return   <code>TRUE</code> if success, <code>FALSE</code> otherwise.
    */
   public static native boolean setWorkingDir(String newWorkingDir);

   /**
    * Replaces UNIX-style and Windows-style file separators with {@link File#separator}.
    * Also expands a file name that begins with "~/" to the user.home system property on UNIX platform.
    *
    * @param    fileName
    *           A file name.
    *           
    * @return   Returns the normalized file name.
    */
   public static String normalizeFileSeparators(String fileName)
   {
      // Check if the fileName starts with a tilde and path separator ('~/')
      if (!isWin && fileName.startsWith("~") && (fileName.length() == 1 || fileName.charAt(1) == '/'))
      {
         // Get the "user.home" property
         String userHome = System.getProperty("user.home");
         
         // Create the expanded components
         fileName = userHome + fileName.substring(1);        
      }
      
      Matcher m = anyPathSep.matcher(fileName);
      return m.replaceAll(isWin ? "\\".concat(File.separator) : File.separator);
   }
   
   /**
    * Find the actual canonical filesystem name that matches the given file or
    * directory, if a case-insensitive match exists.
    *
    * @param    file
    *           The filesystem object to search for.
    * @param    isDir
    *           <code>true</code> if searching for a directory and
    *           <code>false</code> if searching for a file.
    *
    * @return   The filesystem name found or <code>null</code> if no such
    *           object exists.
    */
   public static String getCaseInsensitiveMatch(File file, boolean isDir)
   {
      String result = null;
      
      try
      {
         // if the file is absolute use canonical path, otherwise use regular
         String fileName = file.getCanonicalPath();
         
         // case-insensitive search
         List<File> list = Utils.getCaseInsensitiveFilenameMatches(fileName);
         
         // use the first match that is valid
         for (File next : list)
         {
            if (next.exists())
            {
               if ((!isDir && next.isFile()) || (isDir && next.isDirectory()))
               {
                  result = next.getCanonicalPath();
                  break;
               }
            }
         }
      }
      catch (IOException exc)
      {
         // TODO: add reporting/logging
         
         // file system error, ignore
      }
      
      return result;
   }
   
   /**
    * Deletes the specified file system entity, if necessary using recursion to delete all 
    * contents (if it is a directory) first.  If any error occurs, no exception will be thrown.
    *
    * @param    target
    *           The target file or directory to delete.
    * @param    recursive
    *           Recursively decend through all subdirectories, deleting the entire subtree.
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems occur).
    */
   public static int delete(File target, boolean recursive)
   {
      return deleteWorker(target, recursive);
   }
   
   /**
    * Deletes the specified file system entity, if necessary using recursion
    * to delete all contents (if it is a directory) first.  If any error 
    * occurs, no exception will be thrown.
    *
    * @param    target
    *           The target file or directory to delete.
    * @param    recursive
    *           Recursively decend through all subdirectories, deleting    
    *           the entire subtree.
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems
    *           occur).
    */
   private static int deleteWorker(File target, boolean recursive)   
   {
      try
      {
         String targetPath;
         try
         {
            targetPath = target.getCanonicalPath();
         }
         catch (IOException e)
         {
            return ERR_IO;
         }

         // check the permissions yet before existence check, don't give any clues about the file system
         FileSystemDaemon inst = instance.get();
         if (inst != null && inst.isServerSideFS && !inst.fsRes.canDelete(targetPath))
         {
            return ERR_PERM_DENIED;
         }

         if (!target.exists())
         {
            // we can't delete what isn't there
            return ERR_NO_SUCH_FILE_OR_DIR;
         }

         if (target.isDirectory())
         {
            // delete contents recursively if requested
            if (recursive)
            {
               File[] children = target.listFiles();
               
               if (children == null)
               {
                  return ERR_IO;
               }
               
               for (int i = 0; i < children.length; i++)
               {
                  deleteWorker(children[i], recursive);
               }
            }
            
            // we can't delete a non-empty directory (if recursive was true,
            // then some child failed to delete, if false then the caller
            // shouldn't have expected this to work)
            if (target.list().length > 0)
            {
               return ERR_DIR_NOT_EMPTY;
            }
         }

         // we are a leaf or we are an empty directory, try to delete target
         // (this must occur after all children are processed in the case of
         // a directory)
         boolean result = target.delete();
         
         if (!result)
         {
            // the deletion failed
            return ERR_UNKNOWN;
         }
      }
      
      catch (SecurityException se)
      {
         return ERR_PERM_DENIED;
      }
      
      return ERR_NO_ERROR;
   }
   
   /**
    * Utility method that checks if a path is in a form that is relative to the current
    * directory.
    * 
    * @param    path
    *           The path to check. No call to {@link #normalizeFileSeparators} will be made. It
    *           is assumed this is already normalized.
    * 
    * @return   <code>true</code> if the path is in a form that is relative to current directory
    *           or <code>false</code> otherwise.
    */
   private static boolean isPathRelativeToCurrentDir(String path)
   {
      String sep = File.separator;
      
      // TODO: In the code below (!path.startsWith(":" + sep, 1)) shouldn't we allow for a
      //       case where there is just the drive letter (C:)? When that drive is the same as
      //       the drive for the current directory, then it is a form of relative reference.
      //       But when the current directory is on a different drive, then this result seems
      //       wrong.
      
      // check if the path is relative to the current dir
      return (path.startsWith("." + sep)              ||
              path.startsWith(".." + sep)             || 
              path.equals(".")                        ||
              path.equals("..")                       ||
              (!isWin && !path.startsWith(sep)) ||
              (isWin && (!path.startsWith(sep) && !path.startsWith(":" + sep, 1))));
   }
   
   /**
    * Return file name if it is absolute or exists in the current directory.
    * 
    * @param    caseSens
    *           <code>true</code> if file system is case-sensitive  
    * @param    filename
    *           The file to check. No call to {@link #normalizeFileSeparators} will be made. It
    *           is assumed this is already normalized.
    * @param    allowDir
    *           <code>true</code> if directories are accepted  
    *        
    * @return   The file name if it is absolute or exists in the current directory.
    */
   private static String getAbsoluteOrCurrent(boolean caseSens,
                                              String  filename,
                                              boolean allowDir)
   {
      FileSystemDaemon fsd = instance.get();
      
      if (fsd != null)
      {
         filename = fsd.resolvePath(filename);
      }
      
      File file = new File(filename);
      
      if (file.isAbsolute() || isPathRelativeToCurrentDir(filename))
      {
         return getFileName(caseSens, allowDir, file, filename);
      }
      
      return null;
   }
   
   /**
    * Return the file name if the file is valid in the given path.
    * 
    * @param    caseSens
    *           <code>true</code> if file system is case-sensitive  
    * @param    path
    *           The path to be checked. This will be normalized before processing by calling
    *           {@link #normalizeFileSeparators}.
    * @param    allow
    *           <code>true</code> if directories are accepted.  
    * @param    filename
    *           The file to check. No call to {@link #normalizeFileSeparators} will be made. It
    *           is assumed this is already normalized.
    *        
    * @return   The file name if file is valid in the given path.
    */
   private static String getInPath(boolean caseSens, String path, String filename, boolean allow) 
   {
      if (path == null)
      {
         return null;
      }
      
      FileSystemDaemon fsd = instance.get();
      
      if (fsd != null)
      {
         filename = fsd.resolvePath(filename);
         path = fsd.resolvePath(path);
      }
      
      File file = new File(filename);
      
      // check if path is empty or if it is a file
      if (path.length() != 0 || !file.isFile())
      {
         path = normalizeFileSeparators(path);
         
         // if we have to replace separator also do this for PROPATH paths
         file = new File(path, filename);
      }
      
      return getFileName(caseSens, allow, file, file.getPath());
   }
   
   /**
    * Return the given file name if file/directory exists and matches the given criteria.
    * 
    * @param    caseSens
    *           <code>true</code> if file system is case-sensitive.  
    * @param    allowDir
    *           <code>true</code> if directories are accepted.  
    * @param    file
    *           The file to be checked.
    * @param    filename
    *           The name to return if it exists and matches the rest of the criteria.
    *           No call to {@link #normalizeFileSeparators} will be made. It is
    *           assumed this is already normalized.
    *        
    * @return   file name if file is valid
    */
   private static String getFileName(boolean caseSens, 
                                     boolean allowDir, 
                                     File    file,
                                     String  filename)
   {
      FileSystemDaemon fsd = instance.get();
      try
      {
         if (fsd != null && fsd.isServerSideFS)
         {
            File absFile = file.getAbsoluteFile();
            if (!fsd.fsRes.canRead(absFile.getCanonicalPath()))
            {
               return null;
            }
         }
      }
      catch (IOException e)
      {
         return null;
      }

      return (file.exists() && (file.isFile() || allowDir)) ? filename : caseSens ? null :
         getCaseInsensitiveMatch(file, false);
   }

   /**
    * Constructs a remote stream instance representing a file using a filename, this file or
    * device will be opened for reading or writing based on the given <code>write</code> flag.
    * The current read/write position will be at offset 0 unless in <code>write</code> mode and 
    * the <code>append</code> flag is on (in which case the write position will be at the end of
    * the file).
    * <p>
    * This stream will NOT have paging active.
    *
    * @param    filename
    *           The name of the file or device to open.  Must not be <code>null</code>.
    * @param    write
    *           If <code>true</code>, open in write mode, otherwise open the file for reading.
    * @param    append
    *           In <code>write</code> mode, if <code>true</code>, the current file pointer is
    *           set to the end of the file. Otherwise if in <code>write</code> mode, the file
    *           is truncated to 0 length. This value is ignored in <code>read</code> mode.
    * @param    hasDuplicatedLastLine
    *           Flag to indicate if the last line in the file is duplicate (true for LOB files).
    *
    * @return   The stream id which is used to route operations to the newly created instance.
    *
    * @throws   ErrorConditionException
    *           If the file does not exist OR if the current user does not have the proper rights
    *           to open the file or device.
    */
   @Override
   public int openFileStream(String filename, boolean write, boolean append, boolean hasDuplicatedLastLine)
   throws ErrorConditionException
   {
      boolean isTerminal = false;
      if (filename != null)
      {
         filename = resolvePathName(filename, false);

         String term = filename.toLowerCase();

         if (term.equals("terminal") || term.equals("term"))
         {
            filename = "terminal";
            isTerminal = true;
         }
      }

      StreamHelper streamHelper = OSResourceManager.getStreamHelper();

      if (isTerminal)
      {
         return streamHelper.openTerminalStream();
      }
      else
      {
         // casting works only if streams are not using rpc
         StreamDaemon sd = (StreamDaemon) streamHelper;
         return sd.store(new FileStream(filename,
                                                  write,
                                                  append,
                                                  true,
                                                  null,
                                                  hasDuplicatedLastLine));
      }
   }

   /**
    * Constructs a remote stream instance representing the OsDirStream result of the INPUT 
    * THROUGH OS-DIR (dir-name) statement.
    * <p>
    * This will take the directory and export the list of files, directory and device names
    * and properties.
    *
    * @param    dir
    *           The <code>String</code> path to the directory on which OS-DIR will be applied.
    * @param    noAttrList
    *           <code>true</code> if property indexes will be added to the results.
    *
    * @return   The stream id which is used to route operations to the newly created instance.
    */
   @Override
   public int openDirStream(String dir, boolean noAttrList)
   {
      dir = resolvePathName(dir, true);

      // casting works only if streams are not using rpc
      StreamDaemon sd = (StreamDaemon) OSResourceManager.getStreamHelper();
      return sd.store(new DirStream(dir, noAttrList, null));
   }
   
   /**
    * Check if the file system is case-sensitive
    * 
    * @return <code>true</code> if the file system is case-sensitive
    */
   @Override
   public boolean isCaseSensitive()
   {
      return caseSens;
   }
   
   /**
    * Search the current directory (and PROPATH if it is configured) for a given filename
    * with the specified extensions. 
    * 
    * @param    filename
    *           The file name.
    * @param    caseSensitiveExts
    *           The extensions to be checked if the file system is case-sensitive.
    * @param    caseInsensitiveExts
    *           The extensions to be checked if the file system is case-insensitive.
    * 
    * @return   filename or <code>null</code> if not found
    */
   @Override
   public String searchPathWithExts(String filename,
                                    List<String> caseSensitiveExts,
                                    List<String> caseInsensitiveExts)
   {
      // make sure there is something to search for (the code below needs to be protected)
      if (filename == null)
      {
         return null;
      }
      
      // if the the OS is windows replace all / to \ from the search paths
      String fn = normalizeFileSeparators(filename);
      
      String result = Utils.tryWhileNull(caseSens ? caseSensitiveExts : caseInsensitiveExts,
                              ext -> getAbsoluteOrCurrent(caseSens, fn + ext, false));
      if (result != null)
      {
         return result;
      }
      
      // not found in root or current, try PROPATH entries
      if (paths == null)
      {
         return null;
      }
      
      return Utils.tryWhileNull(Arrays.asList(paths), 
                path -> Utils.tryWhileNull(caseSens ? caseSensitiveExts : caseInsensitiveExts,
                                           ext -> getInPath(caseSens, path, fn + ext, false)) );
   }
   
   /**
    * Access file system details regarding a specific file.
    *
    * @param    name
    *           The relative or absolute filename for which to obtain 
    *           information.
    *
    * @return   The file details if the file was found and can be read or 
    *           default values otherwise.
    */
   public FileInfo accessFileInfo(String name)
   {
      FileInfo info = new FileInfo();
      
      name = resolvePath(name);
      
      info.pathname = search(paths, caseSens, name, true);
      
      // did we find a match?
      if (info.pathname != null)
      {
         // initialize the rest of our data
         File local = new File(info.pathname);

         try
         {
            FileSystemDaemon inst = instance.get();
            FileSystemResource fsr = inst.fsRes;
            boolean isServerSideFS = inst.isServerSideFS;
            String canonPath = local.getCanonicalPath();

            if (!isServerSideFS || fsr.canRead(canonPath))
            {
               info.size = local.length();
               info.fullname = canonPath;

               info.modification = local.lastModified();
               info.creation = FileChecker.getFileCreationTime(info.fullname);

            }

            String primeType = FileChecker.getFileType(info.fullname);
            StringBuilder sb = new StringBuilder(primeType);

            if ((!isServerSideFS || fsr.canRead(info.fullname)) && local.canRead())
               sb.append("R");
            
            try
            {
               if ((!isServerSideFS || fsr.canWrite(info.fullname)) && local.canWrite())
                  sb.append("W");
            }
            catch (SecurityException se)
            {
               // it is OK to not have write access, don't bomb out
            }
            
            info.type = sb.toString();
         }
         catch (Exception exc)
         {
            // we probably aren't allowed to read the file, but there may
            // have been some other failure, reset everything to unknown
            info = new FileInfo();
         }
      }
      
      // the name is always set to the unchanged input value, even if it is
      // invalid
      info.name = name;
      
      return info;
   }
   
   /**
    * Get the comma-separated list of filesystem roots (drive letters on
    * platforms that support the concept, otherwise returns the equivalent
    * of the empty string).
    *
    * @return   The comma-separated list of drive letters or the empty string
    *           if the platform does not support the concept.
    */
   public String getRootList()
   throws IOException
   {
      StringBuilder sb = new StringBuilder("");
      
      File[] drives = File.listRoots();
      
      for (int i = 0; i < drives.length; i++)
      {
         // if this is the 2nd or later iteration, add a comma separator
         if (i > 0)
         {
            sb.append(",");
         }
         
         String name = drives[i].getCanonicalPath();
         int    idx  = name.lastIndexOf(File.separator);
         
         // add the path
         sb.append( idx < 0 ? name : name.substring(0, idx) );
      }
      
      return sb.toString(); 
   }
   
   /**
    * Get the system property (user-defined properties can be specified
    * at the Java command line) associated with the passed key.
    *
    * @return   The system property found or <code>null</code> if the
    *           property does not exist.
    */
   public String getProperty(String key)
   {
      if (key == null || key.length() == 0)
      {
         return null;
      }
      
      // first check the OS for environment variable value
      String res = getEnvironmentValue(key);
      
      return res != null ? res : System.getProperty(key);
   }
   
   /**
    * Get the file separator for the system.
    *
    * @return   The file separator.
    */
   public String getFileSeparator()
   {
      return File.separator;
   }
      
   /**
    * Get the path separator for the system.
    *
    * @return   The path separator.
    */
   public String getPathSeparator()
   {
      return File.pathSeparator;
   }
      
   /**
    * Creates each of the directories requested, if it doesn't exist and all
    * path segments up to the last segment do exist.  If the directory name
    * already exists in the filesystem (even if it is a file), this method
    * just returns (this is the Progress behavior).  If any error occurs, no
    * exception will be thrown, but no subsequent directories in the list
    * will be created.
    *
    * @param    dirs
    *           The list of directories to create.    
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems
    *           occur).
    */
   public int mkdir(String[] dirs)
   {
      try
      {
         for (int i = 0; i < dirs.length; i++)
         {
            // safety first, should not happen
            if (dirs[i] == null)
            {
               // but if it does, return now
               return ERR_UNKNOWN;
            }
            
            dirs[i] = resolvePath(dirs[i]);
            
            String normalizedPath = normalizeFileSeparators(dirs[i]);
            
            File target = new File(normalizedPath);
            
            if (target.exists() || target.getPath().isEmpty())
            {
               // nothing to do (Progress DOES NOT error even when the target is a FILE!)
               continue;
            }
            
            File parent = target.getParentFile();
            
            // is a parent dir specified? (if it is relative, it is not)
            if (parent != null)
            {
               // the parent needs to exist
               if (!parent.exists())
               {
                  boolean found = false;
                  
                  if (!caseSens)
                  {
                     String parentName = getCaseInsensitiveMatch(parent, true);
                     
                     found = parentName != null;
                     
                     if (found)
                     {
                        if (normalizedPath.endsWith(File.separator))
                        {
                           normalizedPath = normalizedPath.substring(0, normalizedPath.length() - 1);
                        }
                        
                        int index = normalizedPath.lastIndexOf(File.separator);
                        
                        if (index != -1)
                        {
                           String targetDir = normalizedPath.substring(index + 1);
                           
                           parent = new File(parentName);
                           
                           target = new File(parent, targetDir);
                           
                           if (target.exists())
                           {
                              continue;
                           }
                        }
                     }
                  }
                  
                  if (!found)
                  {
                     return ERR_NO_SUCH_FILE_OR_DIR;
                  }
               }
               
               // the parent needs to be a directory
               if (!parent.isDirectory())
               {
                  return ERR_NOT_DIR;
               }
            }

            try
            {
               if (isServerSideFS && !fsRes.canCreate(target.getCanonicalPath()))
               {
                  return ERR_PERM_DENIED;
               }
            }
            catch (IOException e)
            {
               // this should not happen
               return ERR_IO;
            }

            // make the directory
            boolean result = target.mkdir();
            
            // did a failure occur?
            if (!result)
            {
               // unknown error 
               return ERR_UNKNOWN;
            }
         }
      }
      
      catch (SecurityException se)
      {
         return ERR_PERM_DENIED;
      }
      
      return ERR_NO_ERROR;
   }
         
   /**
    * Renames (and moves if the path is different) the specified source file
    * to the target filename (and path).  If any error occurs, no
    * exception will be thrown.
    *
    * @param    source
    *           The name of the source file or directory.
    * @param    target
    *           The name of the target file or directory.  If this is a
    *           directory name for a directory that already exists and the
    *           <code>source</code> is a file, then the source file name will
    *           be used as the target file in the directory given. 
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems
    *           occur).
    */
   public int rename(String source, String target)
   {
      if (source == null || target == null)
      {
         return ERR_NO_ERROR;
      }
      
      source = resolvePath(source);
      
      target = resolvePath(target);
      
      File src = new File(normalizeFileSeparators(source));
      File tar = new File(normalizeFileSeparators(target));
      
      try
      {
         // can't rename a non-existing source file or directory
         if (!src.exists())
         {         
            return ERR_NO_SUCH_FILE_OR_DIR;
         }
         
         if (tar.exists())
         {
            if (tar.isDirectory())
            {
               // append the source file or directory name to the target path
               tar = repathSource(src, target);
            }
            else
            {
               return ERR_FILE_EXISTS;
            }
         }

         if (isServerSideFS)
         {
            try
            {
               String srcPath = src.getCanonicalPath();
               String tarPath = tar.getCanonicalPath();
               if (!fsRes.canDelete(srcPath) || !fsRes.canCreate(tarPath))
               {
                  return ERR_PERM_DENIED;
               }
            }
            catch (IOException e)
            {
               return ERR_IO;
            }
         }
         
         // try the rename operation
         boolean result = src.renameTo(tar);
         
         // did a failure occur?
         if (!result)
         {
            // unknown error 
            return ERR_UNKNOWN;
         }
      }
      
      catch (SecurityException se)
      {
         return ERR_PERM_DENIED;
      }      
      
      return ERR_NO_ERROR;
   }
   
   /**
    * Initializes the path for subsequent searches by {@link #searchPath}. 
    *
    * @param    paths
    *           The list of paths to search for files specified by relative
    *           filename.
    * @param    caseSens
    *           <code>true</code> if the legacy system had a case-sensitive
    *           file-system.
    */
   public void setSearchPath(String[] paths, boolean caseSens)
   {
      this.caseSens = caseSens;
      this.paths    = fixPathNames(paths);
   }
   
   /**
    * Gets a copy of its search path.
    * 
    * @return   The list backed by the search path array copy.
    */
   @Override
   public List<String> getSearchPath()
   {
      if (paths != null && paths.length > 0)
      {
         return Arrays.asList(Arrays.copyOf(paths, paths.length));
      }
      else
      {
         return Collections.emptyList();
      }
   }
   
   /**
    * Search for the given file in each directory listed in the path
    * previously specified via {@link #setSearchPath}.  If the file system
    * object exists and is a file, then the filename is returned. If the path
    * contains an empty string or a '.', then the current directory is
    * searched, otherwise only the listed directories are searched.  
    * <p>
    * If the filename is specified as an absolute name, this specific 
    * filename is checked and no path searching is done.
    *
    * @param    filename
    *           The absolute or relative filename for which to search.
    *
    * @return   If the file is found in the current directory, the relative
    *           filename is returned. If the file is absolute or found in any
    *           other path, the full absolute filename is returned.  If the
    *           file is not found or the referenced file system object is not
    *           a file, <code>null</code> is returned.
    */
   public String searchPath(String filename)
   {
      boolean beginsWithDot = filename.startsWith(".");
      
      // the resolved file name can begin with "."
      filename = resolvePath(filename);
      
      return search(paths, caseSens, filename, false, beginsWithDot);
   }
   
   /**
    * Copies (or appends) the specified source file to or over the target
    * file. The mode (copy/overwrite or append) is determined by the
    * <code>overwrite</code> parameter.  If any error occurs, no exception
    * will be thrown.
    *
    * @param    source
    *           The name of the source file.  Must NOT be a directory name.
    *           Must be an existing file.
    * @param    target
    *           The name of the target file.  If this is a directory name,
    *           the source file name will be used as the target file in
    *           the directory given. 
    * @param    overwrite
    *           If <code>true</code> this is a copy, otherwise it is an
    *           append.
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems
    *           occur).
    */
   public int copy(String source, String target, boolean overwrite)
   {
      source = resolvePath(source);
      target = resolvePath(target);
      
      source = normalizeFileSeparators(source);
      target = normalizeFileSeparators(target);
      
      File src = new File(source);
      File tar = new File(target);
      
      try
      {
         // make sure the source file exists
         if (!src.exists())
         {
            return ERR_NO_SUCH_FILE_OR_DIR;
         }
         
         // the source file can't be a directory
         if (src.isDirectory())
         {
            return ERR_IS_DIR;
         }

         String srcPath;
         try
         {
            srcPath = src.getCanonicalPath();
         }
         catch (IOException e)
         {
            return ERR_IO;
         }

         // we need read access to the source
         if (isServerSideFS && !fsRes.canRead(srcPath) || !src.canRead())
         {
            return ERR_PERM_DENIED;
         }
         
         // preprocess the target file
         if (tar.exists())
         {
            if (tar.isDirectory())
            {
               // switch to overwrite mode if not already in that mode
               overwrite = true;
               
               // append the source file name to the target path
               tar = repathSource(src, target);
               
               target = tar.getAbsolutePath();
            }
            
            // re-check because directory processing may have reset the
            // target to a file that doesn't exist
            if (tar.exists())
            {         
               if (tar.isFile())
               {
                  String tarCanPath;
                  try
                  {
                     tarCanPath = tar.getCanonicalPath();
                  }
                  catch (IOException e)
                  {
                     return ERR_IO;
                  }

                  // we need write access to the target (this can only be
                  // called if the file exists)
                  if (isServerSideFS && !fsRes.canWrite(tarCanPath) || !tar.canWrite())
                  {
                     return ERR_PERM_DENIED;
                  }
                  
                  if (overwrite)
                  {
                     // delete the target file
                     tar.delete();
                  }
               }
               else if (tar.isDirectory())
               {
                  // something is wrong and it can't be corrected
                  return ERR_IS_DIR;
               }
            }
         }
         else
         {
            // switch to overwrite mode if not already in that mode, since
            // there is no target file
            overwrite = true;
         }

         if (isServerSideFS)
         {
            try
            {
               String tarPath = tar.getCanonicalPath();

               if (!tar.exists() && !fsRes.canCreate(tarPath))
               {
                  return ERR_PERM_DENIED;
               }

               if (!fsRes.canWrite(tarPath))
               {
                  return ERR_PERM_DENIED;
               }
            }
            catch (IOException e)
            {
               return ERR_IO;
            }
         }

         // best efforts attempt to copy
         boolean result = Utils.copyFile(source, target, 0, !overwrite);
         
         // did a failure occur?
         if (!result)
         {
            // on any failure, remove partial results
            if (tar.exists())
            {
               tar.delete();
            }
            
            // unknown error 
            return ERR_UNKNOWN;
         }
      }
      
      catch (SecurityException se)
      {
         return ERR_PERM_DENIED;
      }
      
      return ERR_NO_ERROR;
   }
   
   /**
    * Deletes the specified file system entity, if necessary using recursion
    * to delete all contents (if it is a directory) first.  If any error 
    * occurs, no exception will be thrown.
    *
    * @param    target
    *           The target file or directory to delete.
    * @param    recursive
    *           Recursively decend through all subdirectories, deleting    
    *           the entire subtree.
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems
    *           occur).
    */
   public int delete(String target, boolean recursive)  
   {
      // safety first, should not happen
      if (target == null)
      {
         // but if it does, return now
         return ERR_UNKNOWN;
      }
      
      target = resolvePath(target);
      
      return deleteWorker(new File(normalizeFileSeparators(target)), recursive);
   }
   
   /**
    * Deletes each of the directories or files specified.  If a non-empty
    * directory name is specified, an error will occur unless the
    * <code>recursive</code> flag is <code>true</code>.  If any error occurs,
    * no exception will be thrown, the deletion process just continues. The
    * last error will be the one returned even if there were prior successful
    * deletes or other failures.
    *
    * @param    list
    *           The list of files and directories to delete.
    * @param    recursive
    *           Recursively decend through all subdirectories, deleting
    *           the entire subtree.
    *
    * @return   The error code (<code>ERR_NO_ERROR</code> if no problems
    *           occur).
    */
   public int delete(String[] list, boolean recursive)
   {
      // set our result code, on error we keep going and the last error is
      // returned, even if some of the deletes succeed (for this reason, if
      // we set the error code at the beginning, then only if everything
      // succeeds will the rc be ERR_NO_ERROR, otherwise the last rc will
      // be accessible --> this is just as Progress does it)
      int rc = ERR_NO_ERROR;
      
      for (int i = 0; i < list.length; i++)
      {
         // safety first, should not happen
         if (list[i] == null)
         {
            // but if it does, return now
            return ERR_UNKNOWN;
         }
         
         list[i] = resolvePath(list[i]);
         
         rc = deleteWorker(new File(normalizeFileSeparators(list[i])), recursive);
      }            
      
      return rc;
   }

   /**
    * Get the file contents from the local file system.
    *
    * @param    filename
    *           Name of the file in the client-side file system (relatively to the current
    *           directory on the client side).
    *           
    * @return   The file's contents as an array of bytes.
    *           
    * @throws   java.io.IOException
    *           If the file cannot not be found or read, or if it is too large.
    */
   @Override
   public byte[] getFileContents(String filename)
   throws IOException
   {
      // safety first, should not happen
      if (filename == null)
      {
         // but if it does, return now
         throw new FileNotFoundException("Filename must not be null.");
      }
      
      filename = resolvePath(filename);
      
      File file = new File(normalizeFileSeparators(filename));
      
      if (!file.exists())
      {
         throw new FileNotFoundException("File " + filename + " does not exist.");
      }
      else if (file.isDirectory())
      {
         throw new FileNotFoundException("File " + filename + " is a directory.");
      }
      else if (file.length() > MAXIMUM_FILE_TRANSFER_SIZE)
      {
         throw new FileNotFoundException("File " + filename + " is too large.");
      }

      String path = file.getCanonicalPath();
      if (isServerSideFS && !fsRes.canRead(path))
      {
         throw new IOException("Permission denied.");
      }

      return Files.readAllBytes(file.toPath());
   }
   
   /**
    * Sets the legacy file system parameters to be used by the client.
    * 
    * @param    fileSeparator
    *           The legacy file separator
    * @param    caseSensitive
    *           The legacy case sensitive mode
    * @param    pathMap
    *           The legacy path map
    * @param    paths
    *           The list of paths to search for files specified by relative
    *           filename.
    * @param    areLegacyPathsNotResolved
    *           The true value of this parameter indicates that legacy path names must be resolved
    *           on the host system, and false value means that legacy path names are not required
    *           to be resolved.
    */
   @Override
   public void setLegacyFileSystemParameters(String              fileSeparator,
                                             boolean             caseSensitive,
                                             Map<String, String> pathMap,
                                             String[]            paths,
                                             boolean             areLegacyPathsNotResolved)
   {
      if (areLegacyPathsNotResolved)
      {
         pathResolver  = Utils.getSafePathResolver(fileSeparator, caseSensitive, pathMap);
      }
      this.caseSens = caseSensitive;
      this.paths    = fixPathNames(paths);
   }

   /**
    * Resolves the given path by the underlined path resolver, otherwise returns the given
    * unresolved value.
    *
    * @param    path
    *           The given path to be resolved
    */
   @Override
   public String resolvePath(String path)
   {
      if (pathResolver == null)
      {
         return path;
      }
      
      return pathResolver.resolvePath(path);
   }

   /**
    * Gets the working directory.
    *
    * @return   The current working directory.
    */
   public String getWorkingDirectory()
   {
      return getWorkingDir();
   }

   /**
    * Sets the working directory via native OS call.
    *
    * @param    newWorkingDir
    *           A new working directory to set.
    */
   public void setWorkingDirectory(String newWorkingDir)
   {
      if (setWorkingDir(newWorkingDir))
      {
         // refresh Java property if directory was really changed
         System.setProperty("user.dir", newWorkingDir);
      }
   }

   /**
    * Gets the name of the directory where temporary files are stored.
    * Corresponds with the <code>SESSION:TEMP-DIRECTORY</code> system handle attribute.
    *
    * @return  Name of temporary directory.
    */
   @Override
   public String getTempDirectory()
   {
      return Utils.getOrCreateDefaultFwdTemporaryDirectory();
   }

   /**
    * Resolves the given legacy path names on the host system and updates them accordingly.
    *  
    * @param    pathNames
    *           The given legacy path names
    * 
    * @return   The array of resolved path names.
    */
   private String[] fixPathNames(String[] pathNames)
   {
      int size = pathNames != null ? pathNames.length : 0;
      
      for(int i = 0; i < size; i++)
      {
         pathNames[i] = resolvePath(pathNames[i]);
      }
      
      return pathNames;
   }
   
   /**
    * Takes the last part of the source filename (removes the path) and
    * appends this to the target directory name.
    *
    * @param    source
    *           The source file or directory. No call to {@link #normalizeFileSeparators}
    *           will be made. It is assumed this is already normalized.
    * @param    target
    *           The target directory (cannot be a file). No call to
    *           {@link #normalizeFileSeparators} will be made. It is assumed this is already
    *           normalized.
    *
    * @return   The object representing the resulting file name. 
    */
   private File repathSource(File source, String target)
   {
      StringBuilder sb = new StringBuilder(target);
      
      // append the path segment separator if needed
      if (!target.endsWith(File.separator))
      {
         sb.append(File.separator);
      }
      
      // append the source file/directory name (remove the path)
      sb.append(source.getName());
      
      // reset the target
      target = sb.toString();
      
      return new File(target);
   }

   /**
    * Tries to resolve the given legacy path name into its host path name, fix it according to
    * the environment file separator and applies case insensitive search if the legacy file system
    * is case insensitive.
    *
    * @param    path
    *           The given legacy path name
    * @param    isDirectory
    *           True iff the given path name represents a directory
    *
    * @return   The transformed path name.
    */
   private String resolvePathName(String path, boolean isDirectory)
   {
      path = resolvePath(path);

      path = FileSystemDaemon.normalizeFileSeparators(path);

      if (!isCaseSensitive())
      {
         File targetFile = new File(path);

         if (targetFile.exists())
         {
            if ((targetFile.isFile() && !isDirectory) || (targetFile.isDirectory() && isDirectory))
            {
               //return path 
               return targetFile.getPath();
            }
         }

         String matched = FileSystemDaemon.getCaseInsensitiveMatch(targetFile, isDirectory);

         if (matched != null)
         {
            return matched;
         }
      }

      return path;
   }
}