RecordLoader.java
/*
** Module : RecordLoader.java
** Abstract : Helper class used for data import
**
** Copyright (c) 2004-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- --JPRM-- -----------------------------------Description-----------------------------------
** 001 ECF 20050819 @22204 Moved initial version from ImportWorker. This class was previously an inner class
** of that top level class. Its base features were abstracted out to be shared
** between database and directory import tasks.
** 002 ECF 20070310 @32338 Added getDMOName() method. Returns the unqualified class name of the
** implementation DMO class associated with the loader.
** 003 ECF 20080107 @36752 Fixed nextRecord() method. Now it properly handles cases where an export data
** file does not contain certain field data (specifically intended to handle
** RECID/ROWID fields, which are not exported).
** 004 GES 20080418 @38174 Moved from a lexer approach to using the equivalent runtime support for the
** Progress IMPORT language stmt. This allows the import process to match the
** behavior of Progress which seems to internally use the IMPORT stmt.
** Note that this allows proper date component ordering, non-ASCII character set
** and other I18N support to work properly.
** 005 CA 20101125 Added method to retrieve the property mapper for a certain property.
** 006 ECF 20181215 Added LOB support.
** 007 OM 20200906 New ORM implementation.
** 008 CA 20200910 Fixed overwriteNullProperty - it was resolving the wrong property for extent
** fields.
** 009 IAS 20201224 Added PropertyMapper by column name lookup
** IAS 20210219 Added full DMO name getter
** TJD 20220504 Java 11 compatibility minor changes
** OM 20220910 Added test for empty records.
** 010 TJD 20240123 Java 17 compatibility updates
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.schema;
import java.io.*;
import java.util.*;
import com.goldencode.p2j.persist.Record;
import com.goldencode.p2j.persist.orm.*;
import com.goldencode.p2j.util.*;
/**
* Helper class which manages all aspects of the mapping between a Progress data export file and
* the record object which corresponds with a particular database table record. The information
* managed by this class includes:
* <ul>
* <li>the record class to be instantiated for each imported record;
* <li>the input data file from which records are read;
* <li>whether the export file reading process should assert the type of each data component it
* reads for a record (for debugging);
* <li>a list of @code PropertyMapper} objects, in the order the mapped data is expected to be
* encountered in the export file for each record (one per column, or one per subscript value
* for extent columns);
* </ul>
* <p>
* An instance of this class is created for each table which needs to be imported. The above
* properties are set as necessary, using the various set, put, and add methods. Once the
* instance is fully configured, {@link #nextRecord(Stream)} should be called repeatedly, until
* there are no more records to process in the input data file.
*/
public abstract class RecordLoader
{
/** Property mappers which define column data */
private final PropertyMapper[] properties;
/** Property mappers which define column data, by property name */
private final Map<String, PropertyMapper> mappersByProp = new HashMap<>();
/** Property mappers which define column data, by column name */
private final Map<String, PropertyMapper> mappersByCol = new HashMap<>();
/** Property mappers which define column native index, by property name */
private final Map<String, Integer> mappersByIndex = new HashMap<>();
/** Class which is instantiated for each import record */
private Class<? extends Record> recordClass = null;
/** Input data file - a Progress export (*.d) file */
private File dataFile = null;
/** Flag to enable assertions of type match for each data field */
private final boolean checkTypes;
/**
* Constructor.
* <p>
* Note: the {@code checkTypes} flag should be set to {@code false} for normal operations.
* It is expensive and should be enabled for debug purposes only.
*
* @param recordClass
* Class which is instantiated for each record imported.
* @param dataFile
* Data input file.
* @param checkTypes
* {@code true} to assert that the data type of each field read matches the expected type.
*/
public RecordLoader(Class<? extends Record> recordClass, File dataFile, boolean checkTypes)
{
this.recordClass = recordClass;
this.dataFile = dataFile;
this.checkTypes = checkTypes;
RecordMeta metadata = Session.getMetadata(recordClass);
PropertyMeta[] metasInNativeOrder = metadata.getPropertyMeta(true);
this.properties = new PropertyMapper[metasInNativeOrder.length];
int index = 0;
PropertyMeta lastMeta = null;
for (int k = 0; k < metasInNativeOrder.length; k++)
{
PropertyMeta meta = metasInNativeOrder[k];
if (lastMeta != meta)
{
lastMeta = meta;
index = 0;
}
else
{
// increment the offset for the next element of the extent variable
index = index + 1;
}
PropertyMapper mapper = new PropertyMapper(meta, meta.getOffset() + index);
properties[k] = mapper;
mappersByProp.putIfAbsent(meta.getName(), mapper); // keep only the first in series
mappersByCol.putIfAbsent(meta.getColumn(), mapper); // keep only the first in series
mappersByIndex.putIfAbsent(meta.getName(), k);
}
}
/**
* Get PropertyMapper by column name.
* @param name
* column name
* @return PropertyMapper for the property with given name.
*/
public PropertyMapper getPropertyMapperByColumn(String name)
{
return mappersByCol.get(name);
}
/**
* Test whether this loader handles empty records (which have no fields / columns except for PK).
*
* @return {@code true} if the records loaded by this instance are empty.
*/
public boolean isRecordEmpty()
{
return properties == null || properties.length == 0;
}
/**
* Overwrites the {@code null} / {@code unknown} values of a specified property and updates it
* to a specified value (if the property is an extent, all indexes are checked and updated).
* This happens for mandatory fields because they were declared as not-null.
*
* @param propertyName
* The name of the property.
* @param dmo
* The record to be updated.
* @param val
* The default value to be used instead.
*/
void overwriteNullProperty(String propertyName, Record dmo, BaseDataType val)
{
PropertyMapper pm = mappersByProp.get(propertyName);
int extent = pm.getPropertyMeta().getExtent();
if (extent == 0)
{
if (pm.isNull(dmo))
{
pm.setValue(dmo, val);
}
}
else
{
int baseOffset = mappersByIndex.get(propertyName);
for (int k = 0; k < extent; k++)
{
pm = properties[baseOffset + k];
if (pm.isNull(dmo))
{
pm.setValue(dmo, val);
}
}
}
}
/**
* Read a single record from the current location in a Progress export file and store its data
* in a bean-like object. Each entry read from the export file represents either a field in the
* current table, or a single value within an extent field. For each record, a new instance of
* the record class associated with this loader is created. The order in which the data is
* tokenized corresponds directly with the order of the {@code PropertyMapper} objects stored
* in this loader instance.
* <p>
* <b>The {@code checkTypes} functionality is not honored here although this would be the
* correct location to add that feature.</b>
*
* @param input
* The export file to be read.
*
* @return A data model object instance containing the data for the next record in the export
* file, or {@code null} if no more records are available.
*
* @throws InstantiationException
* if any error occurs creating a new instance of the DMO class.
* @throws IllegalAccessException
* if the record class, its default constructor, or a setter method which we invoke is
* not accessible.
*/
Record nextRecord(Stream input)
throws ReflectiveOperationException
{
Record record = recordClass.getDeclaredConstructor().newInstance();
record.initialize(null, false);
for (PropertyMapper mapper : properties)
{
// initialize our storage (with an instance of the right type)
BaseDataType value = mapper.createPlaceholder();
// for a property for which we expect to find an exported value in the data file, read
// the next entry and assign it into our storage
if (mapper.isExported())
{
try
{
input.readField(value);
}
catch (EndConditionException ece)
{
// we have encountered the end of the export file
return null;
}
}
// otherwise, assume the value is not present in the exported data and instantiate
// unknown value for the missing datum
else
{
value.setUnknown();
}
processNextValue(mapper, record, mapper.preprocessDatum(value));
}
// notify the stream that the end of the record has been reached
input.resetCurrentLine();
return record;
}
/**
* Process the data value most recently read from the data export file. This default
* implementation simply sets this value into the record object using reflection. Subclasses
* needing more advanced processing should override this method.
*
* @param mapper
* Mapping object associated with the current property.
* @param record
* Java-bean-like object which holds the contents of the record currently being
* loaded.
* @param value
* Data value read from the data export file for the current property. It has
* <i>not</i> yet been stored in the {@code record} object.
*/
protected void processNextValue(PropertyMapper mapper, Record record, BaseDataType value)
{
mapper.setValue(record, value);
}
/**
* Get the full name of the DMO class associated with this record loader.
*
* @return Full DMO class name.
*/
protected String getFullDMOName()
{
return recordClass.getName();
}
/**
* Get the unqualified name of the DMO class associated with this record loader.
*
* @return Unqualified DMO class name.
*/
protected String getDMOName()
{
String dmoName = recordClass.getName();
return dmoName.substring(dmoName.lastIndexOf(".") + 1);
}
/**
* Retrieve the file containing the export data for the current table.
*
* @return Export data file.
*/
File getDataFile()
{
return dataFile;
}
}