FileListFactory.java
/*
** Module : FileListFactory.java
** Abstract : static helpers to create file lists
**
** Copyright (c) 2021-2023, Golden Code Development Corporation.
**
** -#- -I- --Date-- --------------------------------------Description---------------------------------------
** 001 GES 20210707 Initial version supporting a file spec + blacklist.
** GES 20220323 Added explicit whitelist helper and moved common file name handling code here.
** CA 20220412 Added file-set support (either -Z command-line option or p2j.cfg.xml configuration).
** 002 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.util;
import java.io.*;
import java.nio.file.*;
import java.util.*;
import java.util.logging.*;
import java.util.regex.*;
import com.goldencode.p2j.cfg.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.util.logging.*;
import com.goldencode.util.*;
/**
* Static helpers to create file lists.
*/
public class FileListFactory
{
/** Logger */
private static final ConversionStatus LOG = ConversionStatus.get(FileListFactory.class);
/**
* Resolve the file set, by using the initial files (loaded from the reference directory or explicit file
* names), and applying the Include and eXclude operations.
*
* @param fileSet
* The file set.
* @param caseSens
* Flag indicating if the legacy operating system is case-sensitive or not.
*
* @return The resolved file list.
*
* @throws IOException
* In case of problems reading files from disk.
*/
public static FileList processFileSet(FileSet fileSet, boolean caseSens)
throws IOException
{
// step 1. create the reference FileList on which the e(X)clude and (I)nclude operation are applied
ExplicitFileList reference = new ExplicitFileList(caseSens);
// the file and directory references to build the file set on which the Include and eXclude operations
// are applied works at the OS level - this requires OS-level case-(in)sensitivity, as the file list
// is read from the OS.
for (PathReference ref : fileSet.getReferences())
{
String path = ref.getPath();
path = path.replace('/', File.separatorChar);
if (ref.isDirectory())
{
DirectoryReference dir = (DirectoryReference) ref;
File startDir = new File(path);
if (!startDir.exists())
{
throw new FileNotFoundException("Directory " + dir.getPath() + " does not exist.");
}
FileSpecList files = new FileSpecList(startDir, dir.getSpec(), dir.isRecursive(), caseSens);
String[] filenames = resolvePaths(files.listFilenames());
reference.addAll(filenames);
}
else
{
ExplicitFileList l1 = new ExplicitFileList(new String[] { path });
String[] filenames = l1.listFilenames();
if (filenames.length != 1)
{
if (filenames.length == 0)
{
throw new FileNotFoundException("File " + ref.getPath() + " does not exist.");
}
else
{
throw new IOException("More than one file found for path [" + ref.getPath() + "] : " +
Arrays.asList(filenames));
}
}
// resolve OS filenames
filenames = resolvePaths(filenames);
reference.add(filenames[0]);
}
}
// step 2. apply the e(X)clude and (I)nclude operations, in their appearance order
ExplicitFileList defs = new ExplicitFileList(reference);
String cwd = "." + File.separator;
for (FilterOperation oper : fileSet.getOperations())
{
// if the legacy OS is case-insensitive, then the matching addresses this
String filter = oper.getFilter();
// remove the './' prefix as 'resolvePaths' will not emit this
if (filter.equals(cwd))
{
filter = ".";
} else if (filter.startsWith(cwd))
{
filter = filter.substring(cwd.length());
}
filter = StringHelper.convertToRegEx(filter);
Pattern pattern = caseSens
? Pattern.compile(filter)
: Pattern.compile(filter, Pattern.CASE_INSENSITIVE);
String[] targets = reference.listFilenames((target) ->
{
return pattern.matcher(target).matches();
});
// resolve OS filenames
targets = resolvePaths(targets);
if (oper.isInclude())
{
defs.addAll(targets);
}
else
{
defs.removeAll(targets);
}
}
return defs;
}
/**
* Load a file set, from a disk file. Each line on this file follows this syntax:
*
* <ul>
* <li>{@code D <recursive_top_level_dir> <file_spec>} adds a top level dir and file spec which will be
* recursively processed to add 0 or more files into the reference file list.</li>
* <li>{@code N <non_recursive_top_level_dir> <file_spec>} adds a top level dir and file spec which will be
* non-recursively processed to add 0 or more files into the list</li>
* <li>{@code F <absolute_or_relative_file_name>} - explicitly adds a single/individual file into the list
* </li>
* <li>{@code X <exclusion_filter_expression_including_wildcards>} - filter to remove from the list</li>
* <li>{@code I <inclusion_filter_expression_including_wildcards>} - filter to add something removed back
* into the list</li>
* <li>{@code # this is a comment}, a line starting with a # character, representing a comment, which will
* be ignored</li>
* </ul>
*
* @param filename
* The file name from which to load the file set.
*
* @return The loaded file set, or {@code null} in case it could not be loaded.
*/
public static FileSet loadFileSet(String filename)
{
try
{
// read the entire file
List<String> strings = StringHelper.readStringList(filename, FileListFactory::commentLine);
FileSet fileSet = new FileSet();
// 0-based index
int idx = -1;
for (String line : strings)
{
String origLine = line;
idx = idx + 1;
line = line.trim();
if (line.isEmpty())
{
continue;
}
char type = line.charAt(0);
type = Character.toUpperCase(type);
// remove the type
line = line.substring(1);
// the type must be followed by at least a whitespace (space or tab)
if (line.isEmpty() || !(line.charAt(0) == ' ' || line.charAt(0) == '\t'))
{
malformedFileSetLine(origLine, idx, filename);
}
// here the line can be safely trimmed; for D and N types, if the path or spec contains spaces,
// the path or spec must be enclosed in double-quotes
line = line.trim();
boolean recursive = false;
switch (type)
{
case 'D':
// recursive, read 'top level dir' and 'file spec'
recursive = true;
case 'N':
// non-recursive, read 'top level dir' and 'file spec'
// recursive flag is set to true for 'D'
// dir and spec can be included in double-quotes, to allow spaces within these.
// any double-quote inside the dir or spec must be escaped like '\"'
String dir;
String spec;
if (line.startsWith("\""))
{
dir = StringHelper.readQuotedString(line);
if (dir == null)
{
malformedFileSetLine(origLine, idx, filename);
}
line = line.substring(dir.length() + 1).trim();
dir = StringHelper.removeQuotes(dir);
dir = dir.replaceAll("\\\"", "\"");
}
else
{
int sepIdx = line.indexOf(' ');
if (sepIdx <= 0)
{
malformedFileSetLine(origLine, idx, filename);
}
dir = line.substring(0, sepIdx);
line = line.substring(sepIdx + 1).trim();
}
spec = line;
if (spec.startsWith("\"") && spec.endsWith("\""))
{
spec = StringHelper.removeQuotes(spec);
spec = spec.replaceAll("\\\"", "\"");
}
if (spec.isEmpty())
{
malformedFileSetLine(origLine, idx, filename);
}
fileSet.addReference(new DirectoryReference(dir, recursive, spec));
break;
case 'F':
// single file
fileSet.addReference(new FileReference(line));
break;
case 'I':
// include a file spec
fileSet.addOperation(new IncludeFilter(line));
break;
case 'X':
// exclude a file spec
fileSet.addOperation(new ExcludeFilter(line));
break;
default:
throw new IllegalStateException("Unknown [" + type + "] type for line [" + origLine + "] " +
"at row " + idx + " in file " + filename + ".");
}
}
return fileSet;
}
catch (IOException exc)
{
LOG.log(Level.SEVERE, "File definitions processing failed.", exc);
return null;
}
}
/**
* Create an instance using the given inputs.
*
* @param filename
* Text file name containing the list of files from which to create the list.
*
* @return The file list or {@code null} if there is a failure.
*/
public static FileList createExplicit(String filename)
{
FileList list = null;
try
{
List<String> strings = StringHelper.readStringList(filename, FileListFactory::cleanup);
String[] textList = strings.toArray(new String[0]);
list = new ExplicitFileList(textList);
}
catch (IOException exc)
{
LOG.log(Level.SEVERE, "ExplicitFileList creation failed.", exc);
}
return list;
}
/**
* Create an instance using the given inputs.
*
* @param rootPath
* Path to the top level root directory in which files will be listed.
* @param spec
* Regex controlling which files in the root path will be processed.
* @param recurse
* {@code true} to process subdirectories recursively.
* @param cs
* {@code true} to process with case-sensitivity.
* @param blacklist
* The filename of the ignore list.
*
* @return The file list or {@code null} if there is a failure.
*/
public static FileList createBlacklist(String rootPath,
String spec,
boolean recurse,
boolean cs,
String blacklist)
{
FileList list = null;
try
{
List<String> filter = StringHelper.readStringList(blacklist, FileListFactory::cleanup);
list = new FileSpecList(new File(rootPath), spec, recurse, cs, filter);
}
catch (IOException exc)
{
LOG.log(Level.SEVERE, "ERROR: FileSpecList creation failed.", exc);
}
return list;
}
/**
* Given a list of directory paths, resolve them in terms that they will match the exact path as reported
* on the operating system were the conversion is ran.
*
* @param paths
* The paths to resolve.
*
* @return The resolved paths.
*
* @throws IOException
* In case a path can't be resolved.
*/
public static String[] resolvePaths(String[] paths)
throws IOException
{
String[] res = new String[paths.length];
for (int i = 0; i < paths.length; i++)
{
Path path = Paths.get(paths[i]);
path = path.normalize();
Path real = path.toRealPath();
Path base = real.subpath(0, real.getNameCount() - path.getNameCount());
Path resolved = real.subpath(base.getNameCount(), real.getNameCount());
res[i] = resolved.toString();
}
return res;
}
/**
* Raise an {@link IllegalStateException} to report an error when reading a file-set configuration file.
*
* @param line
* The original line.
* @param idx
* The line index in the file.
* @param filename
* The name of the file.
*/
private static void malformedFileSetLine(String line, int idx, String filename)
{
throw new IllegalStateException("Line [" + line + "] at row " + idx + " in file " + filename +
" is malformed.");
}
/**
* Check if the line is a comment line.
*
* @param input
* The line to check.
*
* @return <code>null</code> if the line starts with '#' character.
*/
private static String commentLine(String input)
{
// ignore comments, the # must appear as the first character on the line otherwise it is treated
// as part of a valid filename (# is a valid character in filenames)
return (input.indexOf('#') == 0) ? null : input.trim();
}
/**
* Replace any file system separators with the OS-specific version and honor the comment character.
* If the first character on the line is a hash ({@code #}) character, then the entire line is ignored
* (dropped from the output). A hash character anywhere else in a given line is treated as valid text.
*
* @param input
* Name of the file to be read. Must be relative to the current directory or fully
* qualified.
*
* @return The cleaned text or {@code null} if this is a comment.
*/
private static String cleanup(String input)
{
input = commentLine(input);
if (input == null)
{
return null;
}
input = input.trim();
if (!input.isEmpty())
{
// allow the input file to use linux-style separators; always normalize this with the OS separator
input = input.replace('/', File.separatorChar);
}
return input;
}
}