# FileSpecList.java

    /*
    ** Module   : FileSpecList.java
    ** Abstract : creates a file list based on user defined specifications, optional recursion, and
    **            the current dir or a user-defined start directory
    **
    ** Copyright (c) 2005-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- JPRM-- -----------------------------------Description-------------------------------------
    ** 001 GES 20050125 @19403 Created initial version supporting wildcards, searches of the current
    **                         dir or a user-defined directory and optional recursion.
    ** 002 GES 20050224 @19950 Added lexicographic sorting of the resulting file list.
    ** 003 ECF 20050315 @20341 Added methods to access the properties of the file list object:
    **                         getStartingDirectory, getFileSpecification, and isRecursive.
    ** 004 GES 20050316 @20400 Save off original (unmodified) file spec for use in reporting or debug
    **                         purposes.
    ** 005 GES 20051005 @22973 Changed listImple() to protected instead of private in order to allow
    **                         subclasses to override the list algorithm. Also made a protected
    **                         default constructor.
    ** 006 GES 20060129 @24128 Moved common logic into a new base class and removed this class from
    **                         the parent hierarchy of ExplicitFileList. This makes for a cleaner set
    **                         of interfaces.
    ** 007 ECF 20190218        Added optional blacklist filtering.
    ** 008 CA  20220412        Added file-set support (either -Z command-line option or p2j.cfg.xml configuration).
    ** 009 GBB 20230512        Logging methods replaced by CentralLogger/ConversionStatus.
    ** 010 OM  20230510        Allow unverified directories to be set as [startDir] in import mode.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.util;

    import java.io.*;
    import java.util.*;
    import java.util.regex.*;
    import com.goldencode.p2j.cfg.*;
    import com.goldencode.p2j.util.logging.*;
    import com.goldencode.util.*;

    /**
     * Creates a sorted list of files in the current directory or a user-defined
     * start directory that match a user-defined specification (including
     * wildcards). No directories will be returned in this list. The list is
     * sorted lexicographically.
     * <p>
     * The user-defined file specification is a shell matching pattern which
     * can accept a modified regular expression where '*' matches any number of
     * any character, '?' matches any single character and '[ ]' is a single
     * character match based on the set of defined alternatives. As an example,
     * to match all files, &quot;*&quot; can be used.  Or to match all files
     * that end in '.c' or '.h' one could use &quot;*.[cChH]&quot;.  Any '.'
     * in the pattern will be escaped such that it matches a '.' directly
     * instead of being a single character wildcard match as in standard regular
     * expressions.
     * <p>
     * The caller may optionally ask for the list to be created recursively
     * (all files that match the specification in the start/current directory
     * and all sub-directories).
     * <p>
     * The caller may specify zero or more blacklist filters to be applied to each
     * file found by the primary file specification. Filter specifications support
     * the asterisk ({@code *}) for zero or more wildcard characters, and the
     * question mark ({@code ?}) for exactly one wildcard character.
     * <p>
     * <strong>Important:</strong> note that blacklist filter specifications must
     * EXACTLY match (except wildcard characters) the names of file paths reported
     * by the operating system, relative to the starting directory. This includes
     * any leading characters (e.g., {@code ./} on Linux), or lack thereof. If not,
     * the filters will not exclude files as expected.
     *
     * @author    GES
     */
    public class FileSpecList
    extends FileList
    {
       /** Logger */
       private static final CentralLogger LOG = CentralLogger.get(FileSpecList.class);

       /** The relative or absolute path to the starting directory. */
       private final File startDir;

       /** The regular expression for filtering the file list. */
       private final String fileSpec;

       /** Original unmodified file specification. */
       private final String origFileSpec;

       /** Specifies if matching files in subdirectories will be included. */
       private final boolean recursion;

       /** Regular expression patterns to specify file names to be excluded from the final list */
       private final Pattern[] filterPatterns;

       /**
        * Base constructor to handle most likely cases.  If a <code>null</code>
        * or a non-directory is passed for the starting directory, the current
        * directory will be used.  In addition, if the directory doesn't exist,
        * the current directory will be used.
        * <p>
        * If the file specification is <code>null</code> or an empty string, the
        * file specification will match all filenames.
        *
        * @param    startDir
        *           The starting directory for the file listing or the current
        *           directory if <code>null</code> or if it does not exist.
        * @param    fileSpec
        *           A shell pattern used to filter the list of files.
        *           Defaults to &quot;*&quot; if <code>null</code>
        * @param    recursion
        *           Specifies if matching filenames in all subdirectories should
        *           be included.
        */
       public FileSpecList(File startDir, String fileSpec, boolean recursion)
       {
          this(startDir, fileSpec, recursion, false, null);
       }

       /**
        * Base constructor to handle most likely cases.  If a <code>null</code>
        * or a non-directory is passed for the starting directory, the current
        * directory will be used.  In addition, if the directory doesn't exist,
        * the current directory will be used.
        * <p>
        * If the file specification is <code>null</code> or an empty string, the
        * file specification will match all filenames.
        *
        * @param    startDir
        *           The starting directory for the file listing or the current
        *           directory if <code>null</code> or if it does not exist.
        * @param    fileSpec
        *           A shell pattern used to filter the list of files.
        *           Defaults to &quot;*&quot; if <code>null</code>
        * @param    recursion
        *           Specifies if matching filenames in all subdirectories should
        *           be included.
        * @param    caseSens
        *           Sets the case-sensitivity of the sorting algorithm.
        */
       public FileSpecList(File startDir, String fileSpec, boolean recursion, boolean caseSens)
       {
          this(startDir, fileSpec, recursion, caseSens, null);
       }

       /**
        * Base constructor to handle most likely cases.  If a {@code null} or a non-directory is
        * passed for the starting directory, the current directory will be used.  In addition, if the
        * directory doesn't exist, the current directory will be used.
        * <p>
        * If the file specification is {@code null} or an empty string, the file specification will
        * match all filenames.
        * <p>
        * Filter specifications may overlap; the first match excludes a file from the list. If no
        * filters are specified, this class behaves the same as its parent.
        *
        * @param    startDir
        *           The starting directory for the file listing. Uses the current directory if
        *           {@code null} or if the given directory does not exist or actually specifies a
        *           non-directory.
        * @param    fileSpec
        *           A shell pattern used to filter the list of files. Defaults to &quot;*&quot; if
        *           {@code null}. This filter is inclusive (i.e., only files which pass the filter
        *           can be included in the file list).
        * @param    recursion
        *           Specifies if matching filenames in all subdirectories should be included.
        * @param    caseSens
        *           Sets the case-sensitivity of the sorting algorithm.
        * @param    filters
        *           Zero or more filter specifications of file name patterns to exclude from the
        *           final file list. May be {@code null} or empty, in which case no additional
        *           filtering is performed beyond the primary {@code fileSpec} inclusive filter.
        */
       public FileSpecList(File startDir,
                           String fileSpec,
                           boolean recursion,
                           boolean caseSens,
                           List<String> filters)
       {
          super(caseSens);

          if (startDir != null && (Configuration.isImportMode() || startDir.isDirectory() && startDir.exists()))
          {
             this.startDir = startDir;
          }
          else
          {
             this.startDir = new File(".");
          }

          if (fileSpec != null && fileSpec.length() > 0)
          {
             origFileSpec = fileSpec;
             this.fileSpec = StringHelper.convertToRegEx(fileSpec);
          }
          else
          {
             origFileSpec = "*";
             this.fileSpec = ".*";
          }

          this.recursion = recursion;

          int len = filters != null ? filters.size() : 0;
          filterPatterns = new Pattern[len];

          for (int i = 0; i < len; i++)
          {
             String regex = StringHelper.convertToRegEx(filters.get(i));

             // compile regex patterns, since they will be used multiple times each
             filterPatterns[i] = isCaseSensitive()
                                    ? Pattern.compile(regex)
                                    : Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
          }
       }

       /**
        * Get the starting directory for the match.
        *
        * @return  Starting directory.
        */
       public File getStartDir()
       {
          return startDir;
       }

       /**
        * Get the file specification (modified to be a regular expression) used
        * for the match.
        *
        * @return  File specification.
        */
       public String getFileSpecAsRegEx()
       {
          return fileSpec;
       }

       /**
        * Get the original unmodified file specification used for the match.
        *
        * @return  File specification.
        */
       public String getFileSpec()
       {
          return origFileSpec;
       }

       /**
        * Indicates whether the match is applied recursively into subdirectories.
        *
        * @return  <code>true</code> if match is recursive, else
        *          <code>false</code>.
        */
       public boolean isRecursive()
       {
          return recursion;
       }

       /**
        * Core implementation of the listing algorithm which uses the
        * <code>File</code> class to make a complete list of the directory
        * contents, then this list is used to build a filtered list of just
        * those files that match the user-defined regular expression. Note
        * that if recursion was previously requested (on construction) by the
        * caller, this method will recursively call itself for every directory
        * it encounters in the list.  Only files will be returned in the results
        * list and all results are added to the results list that is passed as
        * a parameter.
        *
        * @param    results
        *           The list to which all matching files must be appended.
        */
       protected void listImpl(List<File> results)
       {
          listImpl(startDir, results);
       }

       /**
        * Core implementation of the listing algorithm which uses the {@code File} class to make a
        * complete list of the directory contents. Then this list is used to build a filtered list
        * of just those files that match the user-defined regular expression. If recursion was
        * requested (on construction) by the caller, this method will recursively call itself for
        * every directory it encounters in the list. If blacklist filters have been specified at
        * construction, they are applied after a file name passes the primary file specification
        * test. Only files will be returned in the results list and all results are added to the
        * results list that is passed as a parameter.
        *
        * @param    dir
        *           The directory in which to list all files.
        * @param    results
        *           The list to which all matching files must be appended.
        */
       protected void listImpl(File dir, List<File> results)
       {
          // we don't use a FilenameFilter here because this would filter out
          // all directories that don't match the regular expression, but this
          // would not be correct, we need to see all directories and we will
          // manually filter the filenames down ourselves
          File[] rawResults  = dir.listFiles();

          // separate file and directory results
          for (int i = 0; i < rawResults.length; i++)
          {
             if (rawResults[i].isDirectory())
             {
                // we only care about directories if recursion is specified
                // otherwise we just skip it
                if (recursion)
                {
                   // recursively call ourselves to process this subdirectory
                   listImpl(rawResults[i], results);
                }
             }
             else
             {
                // only accept files that match the primary regular expression
                File file = rawResults[i];
                Pattern pat = isCaseSensitive()
                                 ? Pattern.compile(fileSpec)
                                 : Pattern.compile(fileSpec, Pattern.CASE_INSENSITIVE);
                if (pat.matcher(file.getName()).matches())
                {
                   // now apply blacklist filters, if any
                   String path = file.getPath();
                   boolean include = true;
                   int len = filterPatterns.length;
                   for (int j = 0; j < len && include; j++)
                   {
                      if (filterPatterns[j].matcher(path).matches())
                      {
                         include = false;
                      }
                   }

                   if (include)
                   {
                      // the file passed all filters; add it to the results list
                      results.add(file);
                   }
                }
             }
          }
       }

       /**
        * Provides a command line interface for an end user to drive and/or test
        * the this class.  The list of resulting filenames is displayed to
        * the user on <code>stdout</code>.
        * <p>
        * Syntax:
        * <pre>
        *    java FileSpecList &lt;directory&gt; &quot;file_spec_pattern&quot;
        *                  [recursion] [ &quot;filter_spec_pattern&quot; [ ... ] ]
        * </pre>
        * Where:
        * <ul>
        *    <li> &lt;directory&gt; is the name of the directory
        *    <li> &quot;file_spec_pattern&quot; is the filename pattern
        *         as used in a shell which can use '*' and '?' as wildcards as
        *         well as a subset of normal regular expression syntax (this
        *         pattern MUST be enclosed in double quotes)
        *    <li> [recursion] is an optional argument that is specified as
        *         'r' or 'R' to request recursion of subdirectories
        *    <li> optional list of filter specifications of file names to exclude,
        *         each of which can use '*' and '?' as wildcards (this pattern
        *         MUST be enclosed in double quotes if it contains wildcards)
        * </ul>
        *
        * @param   args
        *          List of command line arguments.
        */
       public static void main(String[] args)
       {
          String syntax = "Syntax: java FileSpecList <directory> \"file_spec_pattern\" [-R] " +
                          "[ \"filter_spec_pattern\" [ ... ] ]";

          int numArgs = args.length;

          if (numArgs < 2)
          {
             LOG.severe(syntax);
             System.exit(-1);
          }

          File dir = new File(args[0]);
          String filespec = args[1];
          int idx = 2;

          boolean recursion = false;
          if (numArgs > idx && args[idx].equalsIgnoreCase("-r"))
          {
             recursion = true;
             idx++;
          }

          List<String> filters = new ArrayList<>();
          while (idx < numArgs)
          {
             filters.add(args[idx++]);
          }

          try
          {
             FileList flist = new FileSpecList(dir, filespec, recursion, false, filters);
             File list[] = flist.list();

             for ( int i = 0; i < list.length; i++ )
             {
                System.out.println(list[i].toString());
             }
          }
          catch (Exception exc)
          {
             LOG.severe("", exc);
          }
       }
    }