# RecordLoader.java

    /*
    ** Module   : RecordLoader.java
    ** Abstract : Helper class used for data import
    **
    ** Copyright (c) 2004-2024, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
    ** 001 ECF 20050819   @22204 Moved initial version from ImportWorker. This class was previously an inner class
    **                           of that top level class. Its base features were abstracted out to be shared
    **                           between database and directory import tasks.
    ** 002 ECF 20070310   @32338 Added getDMOName() method. Returns the unqualified class name of the
    **                           implementation DMO class associated with the loader.
    ** 003 ECF 20080107   @36752 Fixed nextRecord() method. Now it properly handles cases where an export data
    **                           file does not contain certain field data (specifically intended to handle
    **                           RECID/ROWID fields, which are not exported).
    ** 004 GES 20080418   @38174 Moved from a lexer approach to using the equivalent runtime support for the
    **                           Progress IMPORT language stmt. This allows the import process to match the
    **                           behavior of Progress which seems to internally use the IMPORT stmt.
    **                           Note that this allows proper date component ordering, non-ASCII character set
    **                           and other I18N support to work properly.
    ** 005 CA  20101125          Added method to retrieve the property mapper for a certain property.
    ** 006 ECF 20181215          Added LOB support.
    ** 007 OM  20200906          New ORM implementation.
    ** 008 CA  20200910          Fixed overwriteNullProperty - it was resolving the wrong property for extent
    **                           fields.
    ** 009 IAS 20201224          Added PropertyMapper by column name lookup
    **     IAS 20210219          Added full DMO name getter
    **     TJD 20220504          Java 11 compatibility minor changes
    **     OM  20220910          Added test for empty records.
    ** 010 TJD 20240123          Java 17 compatibility updates
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.schema;

    import java.io.*;
    import java.util.*;
    import com.goldencode.p2j.persist.Record;
    import com.goldencode.p2j.persist.orm.*;
    import com.goldencode.p2j.util.*;

    /**
     * Helper class which manages all aspects of the mapping between a Progress data export file and
     * the record object which corresponds with a particular database table record. The information
     * managed by this class includes:
     * <ul>
     * <li>the record class to be instantiated for each imported record;
     * <li>the input data file from which records are read;
     * <li>whether the export file reading process should assert the type of each data component it
     *     reads for a record (for debugging);
     * <li>a list of @code PropertyMapper} objects, in the order the mapped data is expected to be
     *     encountered in the export file for each record (one per column, or one per subscript value
     *     for extent columns);
     * </ul>
     * <p>
     * An instance of this class is created for each table which needs to be imported. The above
     * properties are set as necessary, using the various set, put, and add methods. Once the
     * instance is fully configured, {@link #nextRecord(Stream)} should be called repeatedly, until
     * there are no more records to process in the input data file.
     */
    public abstract class RecordLoader
    {
       /** Property mappers which define column data */
       private final PropertyMapper[] properties;

       /** Property mappers which define column data, by property name */
       private final Map<String, PropertyMapper> mappersByProp = new HashMap<>();

       /** Property mappers which define column data, by column name */
       private final Map<String, PropertyMapper> mappersByCol = new HashMap<>();

       /** Property mappers which define column native index, by property name */
       private final Map<String, Integer> mappersByIndex = new HashMap<>();

       /** Class which is instantiated for each import record */
       private Class<? extends Record> recordClass = null;

       /** Input data file - a Progress export (*.d) file */
       private File dataFile = null;

       /** Flag to enable assertions of type match for each data field */
       private final boolean checkTypes;

       /**
        * Constructor.
        * <p>
        * Note:  the {@code checkTypes} flag should be set to {@code false} for normal operations.
        *        It is expensive and should be enabled for debug purposes only.
        *
        * @param   recordClass
        *          Class which is instantiated for each record imported.
        * @param   dataFile
        *          Data input file.
        * @param   checkTypes
        *          {@code true} to assert that the data type of each field read matches the expected type.
        */
       public RecordLoader(Class<? extends Record> recordClass, File dataFile, boolean checkTypes)
       {
          this.recordClass = recordClass;
          this.dataFile = dataFile;
          this.checkTypes = checkTypes;

          RecordMeta metadata = Session.getMetadata(recordClass);
          PropertyMeta[] metasInNativeOrder = metadata.getPropertyMeta(true);
          this.properties = new PropertyMapper[metasInNativeOrder.length];
          int index = 0;
          PropertyMeta lastMeta = null;
          for (int k = 0; k < metasInNativeOrder.length; k++)
          {
             PropertyMeta meta = metasInNativeOrder[k];
             if (lastMeta != meta)
             {
                lastMeta = meta;
                index = 0;
             }
             else
             {
                // increment the offset for the next element of the extent variable
                index = index + 1;
             }
             PropertyMapper mapper = new PropertyMapper(meta, meta.getOffset() + index);
             properties[k] = mapper;
             mappersByProp.putIfAbsent(meta.getName(), mapper); // keep only the first in series
             mappersByCol.putIfAbsent(meta.getColumn(), mapper); // keep only the first in series
             mappersByIndex.putIfAbsent(meta.getName(), k);
          }
       }

       /**
        * Get PropertyMapper by column name.
        * @param name
        *        column name
        * @return PropertyMapper for the property with given name.
        */
       public PropertyMapper getPropertyMapperByColumn(String name)
       {
          return mappersByCol.get(name);
       }

       /**
        * Test whether this loader handles empty records (which have no fields / columns except for PK).
        *
        * @return  {@code true} if the records loaded by this instance are empty.
        */
       public boolean isRecordEmpty()
       {
          return properties == null || properties.length == 0;
       }

       /**
        * Overwrites the {@code null} / {@code unknown} values of a specified property and updates it
        * to a specified value (if the property is an extent, all indexes are checked and updated).
        * This happens for mandatory fields because they were declared as not-null.
        *
        * @param   propertyName
        *          The name of the property.
        * @param   dmo
        *          The record to be updated.
        * @param   val
        *          The default value to be used instead.
        */
       void overwriteNullProperty(String propertyName, Record dmo, BaseDataType val)
       {
          PropertyMapper pm = mappersByProp.get(propertyName);
          int extent = pm.getPropertyMeta().getExtent();
          if (extent == 0)
          {
             if (pm.isNull(dmo))
             {
                pm.setValue(dmo, val);
             }
          }
          else
          {
             int baseOffset = mappersByIndex.get(propertyName);
             for (int k = 0; k < extent; k++)
             {
                pm = properties[baseOffset + k];
                if (pm.isNull(dmo))
                {
                   pm.setValue(dmo, val);
                }
             }
          }
       }

       /**
        * Read a single record from the current location in a Progress export file and store its data
        * in a bean-like object. Each entry read from the export file represents either a field in the
        * current table, or a single value within an extent field. For each record, a new instance of
        * the record class associated with this loader is created. The order in which the data is
        * tokenized corresponds directly with the order of the {@code PropertyMapper} objects stored
        * in this loader instance.
        * <p>
        * <b>The {@code checkTypes} functionality is not honored here although this would be the
        * correct location to add that feature.</b>
        *
        * @param   input
        *          The export file to be read.
        *
        * @return  A data model object instance containing the data for the next record in the export
        *          file, or {@code null} if no more records are available.
        *
        * @throws  InstantiationException
        *          if any error occurs creating a new instance of the DMO class.
        * @throws  IllegalAccessException
        *          if the record class, its default constructor, or a setter method which we invoke is
        *          not accessible.
        */
       Record nextRecord(Stream input)
       throws ReflectiveOperationException
       {
          Record record = recordClass.getDeclaredConstructor().newInstance();
          record.initialize(null, false);
          for (PropertyMapper mapper : properties)
          {
             // initialize our storage (with an instance of the right type)
             BaseDataType value = mapper.createPlaceholder();

             // for a property for which we expect to find an exported value in the data file, read
             // the next entry and assign it into our storage
             if (mapper.isExported())
             {
                try
                {
                   input.readField(value);
                }
                catch (EndConditionException ece)
                {
                   // we have encountered the end of the export file
                   return null;
                }
             }
             // otherwise, assume the value is not present in the exported data and instantiate
             // unknown value for the missing datum
             else
             {
                value.setUnknown();
             }

             processNextValue(mapper, record, mapper.preprocessDatum(value));
          }

          // notify the stream that the end of the record has been reached
          input.resetCurrentLine();

          return record;
       }

       /**
        * Process the data value most recently read from the data export file. This default
        * implementation simply sets this value into the record object using reflection. Subclasses
        * needing more advanced processing should override this method.
        *
        * @param   mapper
        *          Mapping object associated with the current property.
        * @param   record
        *          Java-bean-like object which holds the contents of the record currently being
        *          loaded.
        * @param   value
        *          Data value read from the data export file for the current property. It has
        *          <i>not</i> yet been stored in the {@code record} object.
        */
       protected void processNextValue(PropertyMapper mapper, Record record, BaseDataType value)
       {
          mapper.setValue(record, value);
       }

       /**
        * Get the full name of the DMO class associated with this record loader.
        *
        * @return  Full DMO class name.
        */
       protected String getFullDMOName()
       {
          return recordClass.getName();
       }

       /**
        * Get the unqualified name of the DMO class associated with this record loader.
        *
        * @return  Unqualified DMO class name.
        */
       protected String getDMOName()
       {
          String dmoName = recordClass.getName();

          return dmoName.substring(dmoName.lastIndexOf(".") + 1);
       }

       /**
        * Retrieve the file containing the export data for the current table.
        *
        * @return  Export data file.
        */
       File getDataFile()
       {
          return dataFile;
       }
    }