# ReportDriver.java

    /*
    ** Module   : ReportDriver.java
    ** Abstract : drives the pattern engine based on a list of report definitions
    **
    ** Copyright (c) 2007-2023, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
    ** 001 GES 20070514   @33498 Created initial version that provides
    **                           support for command line processing based
    **                           on a list of report definitions.
    ** 002 GES 20070626   @34286 Added support for naming the master report
    **                           index and source file index.
    ** 003 GES 20090424   @41934 Import change.
    ** 004 GES 20090515   @42212 Another import change.
    ** 005 GES 20130203          Rewrite to process all reports in a single pattern
    **                           engine run. This cuts the total time of a reporting
    **                           run down by 80%! As part of this, the input format
    **                           (for defining the reports) has been streamlined and
    **                           the reports are parsed into a list of report
    **                           definition objects.
    ** 006 ECF 20150715          Replace StringBuffer with StringBuilder.
    ** 007 GES 20160311          Added supportLvlExpr parsing and assignment into the report def.
    ** 008 CA  20170118          Added -T option, which is like -S but with a trailing of explicit
    **                           schema AST file names.
    ** 009 ECF 20170505          Retrofit for use with FWD Analytics web application.
    ** 010 GES 20210707          Added ignore list support.
    **     OM  20210907          Added new, much simplified syntax which allows the file list parameters to
    **                           be automatically detected from the configuration file.
    **     OM  20210913          Create list of processed files based on flags read from AST registry.
    **     OM  20210923          Added -P<name> command line option for specifying the configuration profile.
    **     GES 20211221          Fixed abend when using new syntax with no non-option args.
    **     OM  20220405          XmlFilePlugin can load either a file or a stream from application's jar.
    **     GES 20220324          Match method name change.
    **     CA  20220501          Each profile has the same structure as the main 'global' config.  Allow multiple
    **                           profiles to be ran at once, with the conversion switching the state between
    **                           profiles, when a resource (like a file or namespace) is being processed.  Only
    **                           front phase is supported at this time.
    ** 011 GBB 20230512          Logging methods replaced by CentralLogger/ConversionStatus.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.report;

    import com.goldencode.ast.*;
    import com.goldencode.p2j.cfg.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.pattern.*;
    import com.goldencode.p2j.uast.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.util.*;
    import org.xml.sax.*;
    import javax.xml.parsers.*;
    import java.io.*;
    import java.util.*;
    import java.util.logging.*;

    /**
     * Special-purpose driver for the pattern engine to process a list of report definitions against a given set
     * of ASTs. This provides a command line interface, please see {@link #main} for the syntax.
     * <p>
     * The report definitions are stored in an XML file with the following structure:
     * <p>
     * <ul>
     *    <li>list (required root element, contains a list of all reports to create)
     *    <ul>
     *       <li>idxname (optional attribute, contains the base file name of the master report index to be
     *           generated, defaults to "index" which would result in the master report filename being
     *           "index.html", when overridden this value will also be prepended to the source file
     *           cross-reference default filename "source_file_index.html")
     *       <li>report (element, 1 or more, represents a single report)
     *       <ul>
     *          <li>name (required attribute, contains the rule-set file name that creates the associated report)
     *          <li>descr (optional attribute, contains the description of the report)
     *          <li>variable (element, 0 or more, represents a user-replaceable input for the report being run)
     *          <ul>
     *             <li>name (required attribute, the name of the variable being set)
     *             <li>value (required attribute, the value to assign to the variable)
     *          </ul>
     *       </ul>
     *    </ul>
     * </ul>
     * <p>
     * Report definition files have a {@code .rpt} filename suffix and must reside in the pattern path.
     */
    public class ReportDriver
    implements DebugLevels,
               XmlConstants,
               ReportConstants
    {
       /** Default admin password system property key */
       private static final String APP_ADMIN_PASS = "appAdminPass";

       /** Reporting rule-set. */
       private static final String ruleset = "reports/consolidated_reports";

       /** Database mode flag. */
       private static boolean database = false;

       /** Logger */
       private static final ConversionStatus LOG = ConversionStatus.get(ReportDriver.class);

       /**
        * Run the pattern engine with the given input.
        *
        * @param    files
        *           List of ASTs to process.
        * @param    dbg
        *           Debug level.
        * @param    init
        *           Variable initializers to enable.
        */
       public static void patternEngine(FileList[] files, int dbg, Map<String, Object> init)
       throws Exception
       {
          PatternEngine engine = new PatternEngine();
          for (FileList fl : files)
          {
             if (fl != null)
             {
                engine.addAstSpec(fl);
             }
          }
          PatternEngine.setDebugLevel(dbg);
          engine.setReadOnly(true);
          engine.setVariableInitializers(init);
          engine.run(ruleset);
       }

       /**
        * Read the given report definitions and store the list of reports for subsequent processing.
        * Read both the standard reports as well as any user-defined reports that have been exported.
        *
        * @param   rptdef
        *          File name root of the XML file storing the list of report definitions.
        *
        * @return  The list of report definitions.
        *
        * @throws  ConfigurationException
        *          if there is an error loading the project configuration.
        * @throws  ParserConfigurationException
        *          if there is an error configuring the XML parser.
        * @throws  SAXException
        *          if the XML SAX parser encounters an error.
        * @throws  IOException
        *          if there is an error accessing the file.
        */
       private static List<ReportDefinition> loadReports(String rptdef)
       throws ConfigurationException,
              ParserConfigurationException,
              IOException,
              SAXException
       {
          ConfigurationPersistence cfgPers = new ConfigurationPersistence();
          List<ReportDefinition> reports = new ArrayList<>();

          // load standard reports
          File file = ConfigLoader.findConfigurationFile(rptdef, RPT_SUFFIX);
          int typePrimary = cfgPers.readXmlReports(file, reports);
          String userRptBase;

          switch (typePrimary)
          {
             case SRC_CODE_CACHE:
                database = false;
                userRptBase = "code";
                break;
             case SRC_SCHEMA:
                database = true;
                userRptBase = "schema";
                break;
             default:
                throw new RuntimeException("Unknown report type: " + typePrimary);
          }

          // load user-defined reports, if any
          file = new File(Configuration.home() +
                          File.separator +
                          String.format(AUTO_EXPORT_RPT_SPEC, userRptBase));
          int typeUser = cfgPers.readXmlReports(file, reports);

          // sanity: make sure user report type, if any, matches primary report type
          if (typeUser != SRC_UNKNOWN && typeUser != typePrimary)
          {
             throw new RuntimeException("Unexpected report type found in " + file + ": " + typeUser);
          }

          return reports;
       }

       /**
        * Provides a command line interface for generating a list of reports using a list of ASTs and a report
        * definition.
        * <p>
        * Syntax:<br>
        * <pre>
        * {@code java -DP2J_HOME=<home> ReportDriver [options]}
        * </pre>
        * <p>
        * Deprecated syntax:<br>
        * <pre>
        * {@code java -DP2J_HOME=<home> ReportDriver [options] <rptdef> <filelist>}
        * {@code java -DP2J_HOME=<home> ReportDriver -S[options] <rptdef> <directory> <filespec>}
        * {@code java -DP2J_HOME=<home> ReportDriver -T[options] <rptdef> <directory> <filespec> <filelist>}
        * </pre>
        * Where:
        * <ul>
        *   <li>Options
        *   <ul>
        *      <li>{@code Dn} = set debug level 'n' (must be a numeric digit between
        *           {@code MSG_NONE} (0) and {@code MSG_TRACE} (3) inclusive
        *      <li>{@code S}  = use {@code filespecs} instead of an explicit file list
        *      <li>{@code T}  = use {@code filespecs} with a trailing explicit file list (optional)
        *      <li>{@code X}  = use a {@code filespec} plus an {@code ignore_list}
        *      <li>{@code N}  = no recursion in {@code filespec} or {@code ignore_list} mode
        *   </ul>
        *   <li>{@code rptdef} is the name of a report definition file found via the pattern path
        *   <li>explicit file list mode
        *   <ul>
        *      <li>{@code <filelist>} is arbitrary list of absolute and/or relative file names to scan
        *   </ul>
        *   <li>{@code <filespec>} mode
        *   <ul>
        *      <li>{@code <directory>} is the directory to search for files
        *      <li>{@code <filespec>} is the file specification that will be used to create a list of files to
        *          process, should be enclosed in double quotes if any of the wildcard characters '*' or '?'
        *          are used
        *   </ul>
        *   <li>{@code <filespec> + <ignore_list>} mode
        *   <ul>
        *      <li>{@code <directory>} is the directory to search for files
        *      <li>{@code <filespec>} is the file specification that will be used to create a list of files to
        *          process, should be enclosed in double quotes if any of the wildcard characters '*' or '?'
        *          are used
        *      <li>{@code <ignore_list>} is the file name which includes the list of directories/files to ignore
        *   </ul>
        * </ul>
        * <p>
        * For the new syntax form (first) the {@code S}, {@code X} and {@code T} modes are unavailable. In this
        * case the full set of parameters are extracted from the configuration file ({@code p2j.cfg.xml}). This is
        * the recommended form to be used because it is much cleaner.<br>
        * The other forms are prone to errors when passing in the parameters, but they are the solution if a
        * partial report is needed.
        *
        * @param   args
        *          List of command line arguments.
        */
       public static void main(String[] args)
       throws ConfigurationException
       {
          // option defaults
          Options opt = processOptions(args);

          // set the configuration profile
          if (opt.cfgProfile != null)
          {
             Configuration.setDefaultProfiles(opt.cfgProfile);
          }

          if (opt.newSyntax)
          {
             newSyntax(opt);
          }
          else
          {
             oldSyntax(opt, args);
          }
       }

       /**
        * Emit a syntax statement and optional error message to
        * <code>stderr</code>, then exit the process with the specified return
        * code. This method is called from {@link #main} when the user passes
        * command line parameters which are not viable.
        *
        * @param    msg
        *           Optional error message; may be <code>null</code>.
        * @param    rc
        *           Process return code used with <code>System.exit()</code>.
        */
       private static void syntax(String msg, int rc)
       {
          if (msg != null)
          {
             LOG.log(Level.SEVERE, msg);
          }

          String[] staticText =
          {
             "Syntax:",
             "   java -DP2J_HOME=<home> ReportDriver [options]",
             "",
             "Deprecated (but supported) syntax:",
             "   java -DP2J_HOME=<home> ReportDriver   [options] <rptdef> <filelist>",
             "   java -DP2J_HOME=<home> ReportDriver -S[options] <rptdef> <directory> \"<filespec>\"",
             "   java -DP2J_HOME=<home> ReportDriver -T[options] <rptdef> <directory> \"<filespec>\" <filelist>",
             "",
             "Where",
             "   Options:",
             "      Dn    = set DEBUG level 'n' (must be a numeric digit between " + MSG_NONE + " and " +
                            MSG_TRACE + " inclusive)",
             "      S     = use a file SPECIFIER instead of an explicit file list",
             "      T     = use filespecs and a TRAIL of explicit files",
             "      N     = NO recursion in filespec mode or trail mode",
             "      Pname = define the configuration PROFILE",
             "",
             "   rptdef     = the report definition file to use, must be found somewhere in the pattern path",
             "   filelist   = arbitrary list of absolute and/or relative file names to scan (default approach)",
             "   directory  = a relative or absolute path (only used when using filespecs)",
             "   filespec   = file filter specification of the filenames in the given directory to scan.",
             "                Should be enclosed in double quotes if any of the wildcard characters '*'",
             "                or '?' are used (only used when using filespecs)",
          };

          LOG.log(Level.WARNING, String.join(System.lineSeparator(), staticText));

          System.exit(rc);
       }

       /**
        * Read and validates the options sent in command line.
        *
        * @param   args
        *          The command-line arguments.
        *
        * @return  A structure which groups all needed parameters.
        */
       private static Options processOptions(String[] args)
       {
          Options opt = new Options();
          if (args.length == 0)
          {
             return opt;
          }

          // process options

          if (!args[0].startsWith("-"))
          {
             opt.newSyntax = false;
             return opt;
          }

          if (args.length >= 2)
          {
             opt.newSyntax = false;
          }

          opt.idx = 0;

          while (opt.idx < args.length && args[opt.idx].startsWith("-"))
          {
             String opts = args[opt.idx].substring(1).toLowerCase(); // drop the '-' prefix
             opt.idx++;

             if (opts.startsWith("p"))
             {
                // this must be a single option token because the schema configuration profile name might contain
                // characters which interfere with other options
                opt.cfgProfile.add(opts.substring(1)); // drop the 'p'
                continue;
             }

             if (opts.indexOf('s') != -1)
             {
                opt.explicit = false;
                opt.filespec = true;
                opt.newSyntax = false;
             }

             if (opts.indexOf('t') != -1)
             {
                opt.explicit = true;
                opt.filespec = true;
                opt.newSyntax = false;
             }

             if (opts.indexOf('x') != -1)
             {
                opt.explicit = false;
                opt.filespec = false;
                opt.ignore = true;
                opt.newSyntax = false;
             }

             opt.recursion = opts.indexOf('n') == -1;

             int dbg = opts.indexOf('d');

             if (dbg != -1)
             {
                char level = opts.charAt(dbg + 1);

                if (!Character.isDigit(level))
                {
                   syntax("Missing numeric debug level character following D option.", -2);
                }
                else
                {
                   opt.debug = Character.digit(level, 10);

                   if (opt.debug < MSG_NONE || opt.debug > MSG_TRACE)
                   {
                      syntax("Debug level must be between " + MSG_NONE + " and " + MSG_TRACE + ".", -3);
                   }
                }
             }
          }

          return opt;
       }

       /**
        * Parse parameters according to old syntax and execute the report accordingly.
        *
        * @param   opt
        *          The options as they were extracted from the first parameter.
        * @param   args
        *          The full set of parameters from command line.
        */
       private static void oldSyntax(Options opt, String[] args)
       {
          // the deprecated mode requires (mode + directory + filespec) OR (mode and a filelist)
          if (args.length < 2)
          {
             syntax(null, -1);
          }

          // process the report definition
          if (opt.idx >= args.length)
          {
             syntax("Missing report definition parameter.", -4);
          }

          try
          {
             List<ReportDefinition> reports = loadReports(args[opt.idx++]);
             FileList[] files = new FileList[2];

             // at this point idx points to the next arg after the options, this must be one of our various forms
             // of filename input

             if (opt.filespec)
             {
                // there must be exactly 2 args left, unless explicit mode is used, too
                int remArgs = args.length - opt.idx;
                if ((!opt.explicit && remArgs != 2) || (opt.explicit && remArgs < 2))
                {
                   syntax("Missing/incomplete filespec parameter.", -7);
                }

                files[0] = new FileSpecList(new File(args[opt.idx]), args[opt.idx + 1], opt.recursion);
                opt.idx += 2;
             }

             if (opt.explicit)
             {
                // there must be at least 1 arg left
                if (args.length <= opt.idx)
                {
                   if (!opt.filespec)
                   {
                      // mandatory only if filespec was not included
                      syntax("Missing file list parameter.", -6);
                   }
                }
                else
                {
                   int      len      = args.length;
                   String[] filelist = new String[len - opt.idx];

                   System.arraycopy(args, opt.idx, filelist, 0, len - opt.idx);
                   files[1] = new ExplicitFileList(filelist);
                }
             }

             if (opt.ignore)
             {
                // there must be exactly 3 args left
                int remArgs = args.length - opt.idx;

                if (remArgs != 3)
                {
                   syntax("Missing/incomplete filespec + ignore list parameter.", -8);
                }

                files[0] = FileListFactory.createBlacklist(args[opt.idx],
                                                           args[opt.idx + 1],
                                                           opt.recursion,
                                                           false,
                                                           args[opt.idx + 2]);

                if (files[0] == null)
                {
                   syntax("Failure creating ignore list.", -9);
                }
                else
                {
                   // TODO: this won't work for .dict files
                   files[0] = new ExplicitFileList(files[0].withSuffix(".ast"));
                }
             }

             Map<String, Object> vars = new HashMap<>();

             String appAdminPass = System.getProperty(APP_ADMIN_PASS);
             if (appAdminPass != null && appAdminPass.length() > 0)
             {
                vars.put(APP_ADMIN_PASS, '"' + StringHelper.stripEnclosing(appAdminPass, '"') + '"');
             }

             // add a definition for database mode
             vars.put("databaseMode", database);

             // add our reports
             vars.put("reports", reports);

             // create a new pattern engine, configure it and run
             patternEngine(files, opt.debug, vars);
          }
          catch (Exception excpt)
          {
             LOG.log(Level.SEVERE, "", excpt);
          }
       }

       /**
        * Parse parameters according to new syntax and execute the two passes with appropriate parameters.
        *
        * @param   opt
        *          The options as they were extracted from the optional first parameter.
        */
       private static void newSyntax(Options opt)
       {
          // make sure the ast manager is set up and initialized
          String name = Configuration.getParameter("registry");
          AstManager.initialize(new XmlFilePlugin(Configuration.forceHome(name), false));

          // 1. ReportDriver [-TDn] reports/schema *.dict
          String[] dicts = AstManager.get().getFiles(AstGenerator.FLAG_DICT);

          // we need to skip the "standard" schema which contains the _meta structure
          int standardIdx = -1;
          for (int i = 0; i < dicts.length; i++)
          {
             String dict = dicts[i];
             if (dict.endsWith("/standard.dict") || dict.endsWith("\\standard.dict"))
             {
                standardIdx = i;
                break;
             }
          }
          if (standardIdx != -1)
          {
             String[] dicts2 = new String[dicts.length - 1];
             System.arraycopy(dicts, 0, dicts2, 0, standardIdx);
             System.arraycopy(dicts, standardIdx + 1, dicts2, standardIdx, dicts.length - standardIdx - 1);
             dicts = dicts2;
          }

          doPass(opt.debug, "reports/schema", dicts);

          // 2. ReportDriver [-SDn] reports/profile *.ast
          String[] files = AstManager.get().getFiles(AstManager.FLAG_ORIGINAL);
          for (int i = 0; i < files.length; i++)
          {
             files[i] += ".ast";

          }
          doPass(opt.debug, "reports/profile", files);
       }

       /**
        * Does one of the passes.
        * @param   debugLevel
        *          The debug level to be used.
        * @param   rptDef
        *          The report definition.
        * @param   files
        *          The list of files to be processed.
        */
       private static void doPass(int debugLevel, String rptDef, String[] files)
       {
          try
          {
             List<ReportDefinition> reports = loadReports(rptDef);
             FileList[] fileLists = new FileList[1];
             fileLists[0] = new ExplicitFileList(files);
             Map<String, Object> vars = new HashMap<>();

             String appAdminPass = System.getProperty(APP_ADMIN_PASS);
             if (appAdminPass != null && appAdminPass.length() > 0)
             {
                vars.put(APP_ADMIN_PASS, '"' + StringHelper.stripEnclosing(appAdminPass, '"') + '"');
             }

             // add a definition for database mode
             vars.put("databaseMode", database);

             // add our reports
             vars.put("reports", reports);

             // create a new pattern engine, configure it and run
             patternEngine(fileLists, debugLevel, vars);
          }
          catch (Exception excpt)
          {
             LOG.log(Level.SEVERE, "", excpt);
          }
       }

       /**
        * A structure grouping all options and additional information extracted from the first entry of the
        * command-line parameters.
        */
       private static class Options
       {
          /** Do the arguments respect the newer, simpler syntax? */
          boolean newSyntax = true;

          /** The command line contains explicit file names. */
          boolean explicit = true;

          /** The command line contains filespecs. */
          boolean filespec = false;

          /** The command line contains filespec plus an ignore-list. */
          boolean ignore = false;

          /** The optional database configuration profile. */
          List<String> cfgProfile = new ArrayList<>();

          /** Allow recursion in filespec mode or trail mode. */
          boolean recursion = true;

          /** The debug level. */
          int debug = MSG_STATUS;

          /** The index of the next parameter to be processed. */
          int idx = 0;
       }
    }