# FileListFactory.java

    /*
    ** Module   : FileListFactory.java
    ** Abstract : static helpers to create file lists
    **
    ** Copyright (c) 2021-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --------------------------------------Description---------------------------------------
    ** 001 GES 20210707 Initial version supporting a file spec + blacklist.
    **     GES 20220323 Added explicit whitelist helper and moved common file name handling code here.
    **     CA  20220412 Added file-set support (either -Z command-line option or p2j.cfg.xml configuration).
    ** 002 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util;

    import java.io.*;
    import java.nio.file.*;
    import java.util.*;
    import java.util.logging.*;
    import java.util.regex.*;

    import com.goldencode.p2j.cfg.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.util.logging.*;
    import com.goldencode.util.*;

    /**
     * Static helpers to create file lists.
     */
    public class FileListFactory
    {
       /** Logger */
       private static final ConversionStatus LOG = ConversionStatus.get(FileListFactory.class);

       /**
        * Resolve the file set, by using the initial files (loaded from the reference directory or explicit file
        * names), and applying the Include and eXclude operations.
        *
        * @param    fileSet
        *           The file set.
        * @param    caseSens
        *           Flag indicating if the legacy operating system is case-sensitive or not.
        *
        * @return   The resolved file list.
        *
        * @throws   IOException
        *           In case of problems reading files from disk.
        */
       public static FileList processFileSet(FileSet fileSet, boolean caseSens)
       throws IOException
       {
          // step 1. create the reference FileList on which the e(X)clude and (I)nclude operation are applied
          ExplicitFileList reference = new ExplicitFileList(caseSens);

          // the file and directory references to build the file set on which the Include and eXclude operations
          // are applied works at the OS level - this requires OS-level case-(in)sensitivity, as the file list
          // is read from the OS.

          for (PathReference ref : fileSet.getReferences())
          {
             String path = ref.getPath();
             path = path.replace('/', File.separatorChar);

             if (ref.isDirectory())
             {
                DirectoryReference dir = (DirectoryReference) ref;
                File startDir = new File(path);
                if (!startDir.exists())
                {
                   throw new FileNotFoundException("Directory " + dir.getPath() + " does not exist.");
                }

                FileSpecList files = new FileSpecList(startDir, dir.getSpec(), dir.isRecursive(), caseSens);
                String[] filenames = resolvePaths(files.listFilenames());
                reference.addAll(filenames);
             }
             else
             {
                ExplicitFileList l1 = new ExplicitFileList(new String[] { path });
                String[] filenames = l1.listFilenames();
                if (filenames.length != 1)
                {
                   if (filenames.length == 0)
                   {
                      throw new FileNotFoundException("File " + ref.getPath() + " does not exist.");
                   }
                   else
                   {
                      throw new IOException("More than one file found for path [" + ref.getPath() + "] : " +
                                            Arrays.asList(filenames));
                   }
                }

                // resolve OS filenames
                filenames = resolvePaths(filenames);
                reference.add(filenames[0]);
             }
          }

          // step 2. apply the e(X)clude and (I)nclude operations, in their appearance order
          ExplicitFileList defs = new ExplicitFileList(reference);

          String cwd = "." + File.separator;

          for (FilterOperation oper : fileSet.getOperations())
          {
             // if the legacy OS is case-insensitive, then the matching addresses this
             String filter = oper.getFilter();
             // remove the './' prefix as 'resolvePaths' will not emit this
             if (filter.equals(cwd))
             {
                filter = ".";
             } else if (filter.startsWith(cwd))
             {
                filter = filter.substring(cwd.length());
             }

             filter = StringHelper.convertToRegEx(filter);
             Pattern pattern = caseSens
                                  ? Pattern.compile(filter)
                                  : Pattern.compile(filter, Pattern.CASE_INSENSITIVE);

             String[] targets = reference.listFilenames((target) ->
             {
                return pattern.matcher(target).matches();
             });

             // resolve OS filenames
             targets = resolvePaths(targets);

             if (oper.isInclude())
             {
                defs.addAll(targets);
             }
             else
             {
                defs.removeAll(targets);
             }
          }

          return defs;
       }

       /**
        * Load a file set, from a disk file.  Each line on this file follows this syntax:
        *
        * <ul>
        * <li>{@code D <recursive_top_level_dir> <file_spec>} adds a top level dir and file spec which will be
        *     recursively processed to add 0 or more files into the reference file list.</li>
        * <li>{@code N <non_recursive_top_level_dir> <file_spec>} adds a top level dir and file spec which will be
        *     non-recursively processed to add 0 or more files into the list</li>
        * <li>{@code F <absolute_or_relative_file_name>} - explicitly adds a single/individual file into the list
        * </li>
        * <li>{@code X <exclusion_filter_expression_including_wildcards>} - filter to remove from the list</li>
        * <li>{@code I <inclusion_filter_expression_including_wildcards>} - filter to add something removed back
        *     into the list</li>
        * <li>{@code # this is a comment}, a line starting with a # character, representing a comment, which will
        *     be ignored</li>
        * </ul>
        *
        * @param    filename
        *           The file name from which to load the file set.
        *
        * @return   The loaded file set, or {@code null} in case it could not be loaded.
        */
       public static FileSet loadFileSet(String filename)
       {
          try
          {
             // read the entire file
             List<String> strings = StringHelper.readStringList(filename, FileListFactory::commentLine);

             FileSet fileSet = new FileSet();

             // 0-based index
             int idx = -1;
             for (String line : strings)
             {
                String origLine = line;

                idx = idx + 1;
                line = line.trim();
                if (line.isEmpty())
                {
                   continue;
                }

                char type = line.charAt(0);
                type = Character.toUpperCase(type);

                // remove the type
                line = line.substring(1);

                // the type must be followed by at least a whitespace (space or tab)
                if (line.isEmpty() || !(line.charAt(0) == ' ' || line.charAt(0) == '\t'))
                {
                   malformedFileSetLine(origLine, idx, filename);
                }

                // here the line can be safely trimmed; for D and N types, if the path or spec contains spaces,
                // the path or spec must be enclosed in double-quotes
                line = line.trim();

                boolean recursive = false;
                switch (type)
                {
                   case 'D':
                      // recursive, read 'top level dir' and 'file spec'
                      recursive = true;
                   case 'N':
                      // non-recursive, read 'top level dir' and 'file spec'
                      // recursive flag is set to true for 'D'

                      // dir and spec can be included in double-quotes, to allow spaces within these.
                      // any double-quote inside the dir or spec must be escaped like '\"'
                      String dir;
                      String spec;
                      if (line.startsWith("\""))
                      {
                         dir = StringHelper.readQuotedString(line);
                         if (dir == null)
                         {
                            malformedFileSetLine(origLine, idx, filename);
                         }

                         line = line.substring(dir.length() + 1).trim();

                         dir = StringHelper.removeQuotes(dir);
                         dir = dir.replaceAll("\\\"", "\"");
                      }
                      else
                      {
                         int sepIdx = line.indexOf(' ');
                         if (sepIdx <= 0)
                         {
                            malformedFileSetLine(origLine, idx, filename);
                         }

                         dir = line.substring(0, sepIdx);
                         line = line.substring(sepIdx + 1).trim();
                      }

                      spec = line;
                      if (spec.startsWith("\"") && spec.endsWith("\""))
                      {
                         spec = StringHelper.removeQuotes(spec);
                         spec = spec.replaceAll("\\\"", "\"");
                      }
                      if (spec.isEmpty())
                      {
                         malformedFileSetLine(origLine, idx, filename);
                      }

                      fileSet.addReference(new DirectoryReference(dir, recursive, spec));
                      break;

                   case 'F':
                      // single file
                      fileSet.addReference(new FileReference(line));
                      break;

                   case 'I':
                      // include a file spec
                      fileSet.addOperation(new IncludeFilter(line));
                      break;
                   case 'X':
                      // exclude a file spec
                      fileSet.addOperation(new ExcludeFilter(line));
                      break;

                   default:
                      throw new IllegalStateException("Unknown [" + type + "] type for line [" + origLine + "] " +
                                                      "at row " + idx + " in file " + filename + ".");
                }
             }

             return fileSet;
          }
          catch (IOException exc)
          {
             LOG.log(Level.SEVERE, "File definitions processing failed.", exc);

             return null;
          }
       }

       /**
        * Create an instance using the given inputs.
        *
        * @param    filename
        *           Text file name containing the list of files from which to create the list.
        *
        * @return   The file list or {@code null} if there is a failure.
        */
       public static FileList createExplicit(String filename)
       {
          FileList list = null;

          try
          {
             List<String> strings = StringHelper.readStringList(filename, FileListFactory::cleanup);
             String[] textList = strings.toArray(new String[0]);
             list = new ExplicitFileList(textList);
          }
          catch (IOException exc)
          {
             LOG.log(Level.SEVERE, "ExplicitFileList creation failed.", exc);
          }

          return list;
       }

       /**
        * Create an instance using the given inputs.
        *
        * @param    rootPath
        *           Path to the top level root directory in which files will be listed.
        * @param    spec
        *           Regex controlling which files in the root path will be processed.
        * @param    recurse
        *           {@code true} to process subdirectories recursively.
        * @param    cs
        *           {@code true} to process with case-sensitivity.
        * @param    blacklist
        *           The filename of the ignore list.
        *
        * @return   The file list or {@code null} if there is a failure.
        */
       public static FileList createBlacklist(String        rootPath,
                                              String        spec,
                                              boolean       recurse,
                                              boolean       cs,
                                              String        blacklist)
       {
          FileList list = null;

          try
          {
             List<String> filter = StringHelper.readStringList(blacklist, FileListFactory::cleanup);

             list = new FileSpecList(new File(rootPath), spec, recurse, cs,  filter);
          }
          catch (IOException exc)
          {
             LOG.log(Level.SEVERE, "ERROR: FileSpecList creation failed.", exc);
          }

          return list;
       }

       /**
        * Given a list of directory paths, resolve them in terms that they will match the exact path as reported
        * on the operating system were the conversion is ran.
        *
        * @param    paths
        *           The paths to resolve.
        *
        * @return   The resolved paths.
        *
        * @throws   IOException
        *           In case a path can't be resolved.
        */
       public static String[] resolvePaths(String[] paths)
       throws IOException
       {
          String[] res = new String[paths.length];

          for (int i = 0; i < paths.length; i++)
          {
             Path path = Paths.get(paths[i]);
             path = path.normalize();

             Path real = path.toRealPath();
             Path base = real.subpath(0, real.getNameCount() - path.getNameCount());
             Path resolved = real.subpath(base.getNameCount(), real.getNameCount());

             res[i] = resolved.toString();
          }

          return res;
       }

       /**
        * Raise an {@link IllegalStateException} to report an error when reading a file-set configuration file.
        *
        * @param    line
        *           The original line.
        * @param    idx
        *           The line index in the file.
        * @param    filename
        *           The name of the file.
        */
       private static void malformedFileSetLine(String line, int idx, String filename)
       {
          throw new IllegalStateException("Line [" + line + "] at row " + idx + " in file " + filename +
                                          " is malformed.");
       }

       /**
        * Check if the line is a comment line.
        *
        * @param    input
        *           The line to check.
        *
        * @return   <code>null</code> if the line starts with '#' character.
        */
       private static String commentLine(String input)
       {
          // ignore comments, the # must appear as the first character on the line otherwise it is treated
          // as part of a valid filename (# is a valid character in filenames)
          return (input.indexOf('#') == 0) ? null : input.trim();
       }

       /**
        * Replace any file system separators with the OS-specific version and honor the comment character.
        * If the first character on the line is a hash ({@code #}) character, then the entire line is ignored
        * (dropped from the output).  A hash character anywhere else in a given line is treated as valid text.
        *
        * @param    input
        *           Name of the file to be read. Must be relative to the current directory or fully
        *           qualified.
        *
        * @return   The cleaned text or {@code null} if this is a comment.
        */
       private static String cleanup(String input)
       {
          input = commentLine(input);
          if (input == null)
          {
             return null;
          }

          input = input.trim();

          if (!input.isEmpty())
          {
             // allow the input file to use linux-style separators; always normalize this with the OS separator
             input = input.replace('/', File.separatorChar);
          }

          return input;
       }
    }