Importer.java
/*
** Module : Importer.java
** Abstract : Base class for readind JSON/XML content into a dataset/temp-table.
**
** Copyright (c) 2023-2024, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
** 001 IAS 20230524 Created initial version.
** 002 CA 20230724 Further reduce context-local usage.
** 003 DDF 20240517 Data relations should be SELECTION by default.
** 004 CA 20240523 Fixed DATASET:WRITE-/READ-XML when before tables or (in)active relations are involved.
** 005 TJD 20240123 Java 17 compatibility updates
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.persist.serial;
import static com.goldencode.p2j.persist.serial.Util.STR_ROW_CREATED;
import static com.goldencode.p2j.persist.serial.Util.STR_ROW_DELETED;
import static com.goldencode.p2j.persist.serial.Util.STR_ROW_MODIFIED;
import java.lang.reflect.*;
import java.util.*;
import java.util.function.*;
import org.apache.commons.lang3.tuple.*;
import com.goldencode.p2j.convert.*;
import com.goldencode.p2j.directory.Base64;
import com.goldencode.p2j.persist.*;
import com.goldencode.p2j.persist.Record;
import com.goldencode.p2j.persist.lock.*;
import com.goldencode.p2j.persist.serial.Importer.ImportContext;
import com.goldencode.p2j.persist.serial.JsonImport.*;
import com.goldencode.p2j.persist.serial.SerializeOptions.*;
import com.goldencode.p2j.util.*;
import com.goldencode.p2j.util.logging.*;
/**
* Base class for readind JSON/XML content into a dataset/temp-table.
*/
public abstract class Importer
{
/** Logger. */
private static final CentralLogger LOG = CentralLogger.get(Importer.class);
/** Import context */
protected ImportContext ctx = new ImportContext();
/** Import context */
protected final Stack<ImportContext> savedCtx = new Stack<>();
/** Data source which normalizes access of various media to an input stream. */
protected final SourceData source;
/** Read mode to determine how records are stored and how non-unique records are handled */
protected final Read readMode;
/** Current dataset, if one is being processed. */
protected DataSet ds = null;
/** Child fields for NESTED relations with FOREIGN-KEY-HIDDEN = true per child table name*/
protected Map<String, Pair<String[], String[]>> childFieldsByChild = new HashMap<>();
/** {@code true} when reading a before-table image. */
protected boolean readingBefore = false;
/** Associates the {@code prod:id} to their record's {@code rowid}.*/
protected Map<String, rowid> peerMapping = null;
/** Helper for setting a {@code Record} into a buffer, avoiding direct access to private method. */
protected BiConsumer<RecordBuffer, Record> loader = null;
/**
* The set of temp-table builders for a dataset. They are mapped by their names.
*/
protected final Map<String, AbstractTempTable> tables = new LinkedHashMap<>();
/** Mapping of tables with before-table names. */
protected final Map<String, String> b4tables = new HashMap<>();
/** Inferred DATA-RELATIONs data */
protected final List<Pair<String, String>> inferredRelations = new ArrayList<>();
/** The {@link BufferManager} instance. */
protected final BufferManager bufMan = BufferManager.get();
/**
* C'tor
*
* @param source
* XML data source which normalizes access of various media to an input stream.
* @param readMode
* Read mode to determine how records are stored and how non-unique records are
* handled. Not currently honored, except temp-table records are deleted if mode
* is EMPTY.
*/
public Importer(SourceData source, String readMode)
{
super();
this.source = source;
this.readMode = (readMode == null) ? Read.MERGE : Read.valueOf(readMode.toUpperCase());
}
/**
* Get an inferred type of the value.
*
* @param val
* The value to be analyzed.
*
* @return an inferred type of the value.
*
* @throws PersistenceException
* on invalid value type
*/
protected abstract ParmType typeOf(Object val)
throws PersistenceException;
/**
* Collect relations with FOREIGN-KEY-HIDDEN = <code>true></code>
*
* @param ds
* DATASET to be analyzed
*/
protected void processHiddenFk(DataSet ds)
{
ds.getRelations().filter(r -> r.getForeignKeyHidden().booleanValue()).
forEach(r -> {
childFieldsByChild.put(r.getChildTable(),
Pair.of(r.getRelationFields(true), r.getRelationFields(false)));
});
}
/**
* Initialize values of the fields to be taken from the parent (for FOREIGN-KEY-HIDDEN support).
*
* @param name
* The name of the child table.
*
* @return <code>true</code> on success.
*/
protected boolean setParentValues(String name)
{
ctx.parentValues = new HashMap<>();
if (ds.getBuffers().isEmpty()) // empty dynamic dataset
{
String link = savedCtx.peek().ttName;
if (link != null)
{
ctx.parentValues.putIfAbsent(link + "_id",
((BufferImpl)savedCtx.peek().tt.defaultBufferHandleNative()).recordID());
}
return true;
}
Pair<String[], String[]> rfields = childFieldsByChild.get(name);
if (rfields != null)
{
String[] pfields = rfields.getLeft();
String[] cfields = rfields.getRight();
for(int nf = 0; nf < pfields.length; nf++)
{
TempTableSchema.Column column = ctx.schema.getColumn(pfields[nf]);
if (column == null)
{
// log "impossible" situation?
return false;
}
try
{
ctx.parentValues.put(cfields[nf], column.getGetter().invoke(ctx.proxy));
}
catch (IllegalAccessException | InvocationTargetException e)
{
LOG.severe("", e);
return false;
}
}
}
return true;
}
/**
* Populate fields from the parent table (for FOREIGN-KEY-HIDDEN support).
*
* @throws PersistenceException
* on database error.
*/
protected void fillFromParent()
throws PersistenceException
{
TempTableSchema.Column column;
for (Map.Entry<String, Object> e: ctx.parentValues.entrySet())
{
String fname = e.getKey();
Object val = e.getValue();
column = getColumn(fname);
setField(column, fname, val == null ? null : val.toString(), false);
}
}
/**
* Persist the read values as a new record.
*
* @param readValues
* The map containing the values of the fields mapped by their name.
* @param addLinkToParent
* Flag indicating that a link to the parent must be added.
*
* @return <code>true</code> on success.
*
* @throws PersistenceException
* in the event of a persistebnce error.
*/
protected boolean persist(FieldMaker maker, Map<String, Object> readValues, boolean addLinkToParent)
throws PersistenceException
{
if (!ctx.tt._prepared())
{
for (Map.Entry<String, Object> m: readValues.entrySet())
{
maker.addField((TempTableBuilder)ctx.tt, m.getKey(), m.getValue());
}
if (addLinkToParent)
{
String link = savedCtx.peek().ttName;
if (link != null)
{
recid rid = ((BufferImpl)savedCtx.peek().tt.defaultBufferHandleNative()).recordID();
String linkId = link + "_id";
((TempTableBuilder)ctx.tt).addNewField(new character(linkId),
new character(ParmType.RECID.toString()),
null, null, null, null, null, null, null, null,
true, false);
readValues.putIfAbsent(linkId, rid);
}
}
if (!ctx.tt.tempTablePrepare(ctx.ttName).booleanValue())
{
return false;
}
}
tables.put(ctx.ttName, ctx.tt);
BufferImpl buffer = (BufferImpl) ctx.tt.defaultBufferHandleNative();
ctx.setBuffer(buffer.buffer(), false);
return populateRecord(readValues);
}
/**
* Populate record with values from JSON/XML.
*
* @param readValues
* The values read from JSON/XML mapped by their field name.
*
* @return <code>true</code> on success.
*
* @throws PersistenceException
* in the event of a persistebnce error.
*/
protected boolean populateRecord(Map<String, Object> readValues)
throws PersistenceException
{
boolean rc = true;
ctx.proxy.create();
RecordBuffer.startBatch(bufMan, true);
try
{
for (Map.Entry<String, Object> m: readValues.entrySet())
{
String name = m.getKey();
Object val = m.getValue();
TempTableSchema.Column column = getColumn(name);
if (column == null)
{
continue;
}
if (val instanceof List)
{
List<Object> list = (List<Object>) val;
for (Object o : list)
{
setField(column, name, o == null ? null : o.toString(), true);
}
}
else
{
setField(column, name, val == null ? null : val.toString(), false);
}
}
}
finally
{
ctx.extentTracker.reset();
ErrorManager.ErrorHelper eh = ErrorManager.getErrorHelper();
boolean wm = eh.isWarningMode();
if (!wm)
{
eh.setWarningMode(true);
}
int validationErrors = RecordBuffer.endBatch(bufMan, false);
if (!wm)
{
eh.setWarningMode(false);
}
if (ErrorManager.isPendingError() || validationErrors != 0)
{
((BufferImpl) ctx.proxy).drop();
ErrorManager.recordOrShowError(15366, true, ctx.schema.getName(), "");
// Unable to update indexes for table '<name>'
rc = false;
}
}
readValues.clear();
return rc;
}
/**
* Set the value of a single field in the temp-table record currently being read from the JSON/XML
* content.
*
* @param column
* Schema information for the associated column.
* @param name
* Element or attribute name associated with serialized JSON/XML data for a temp-table
* column.
* @param value
* Data value read from JSON/XML content. Set to {@code null} to represent unknown value.
*
* @throws PersistenceException
* if there is an error storing data in the temp-table.
*/
protected void setField(TempTableSchema.Column column, String name, String value)
throws PersistenceException
{
Integer index = ctx.extentTracker.getNext(name);
setField(column, name, value, (index != null) ? index : -1);
}
/**
* Set the value of a single field in the temp-table record currently being read from the JSON/XML content.
*
* @param column
* Schema information for the associated column.
* @param name
* Element or attribute name associated with serialized JSON/XML data for a temp-table column.
* @param value
* Data value read from JSON content. Set to {@code null} to represent unknown value.
* @param extent
* {@code true} to indicate an extent field; {@code false} if scalar.
*
* @throws PersistenceException
* if there is an error storing data in the temp-table.
*/
protected void setField(TempTableSchema.Column column, String name, String value, boolean extent)
throws PersistenceException
{
setField(column, name, value, extent ? ctx.extentTracker.getNext(name) : -1);
}
/**
* Set the value of a single field in the temp-table record currently being read from the JSON/XML content.
*
* @param column
* Schema information for the associated column.
* @param name
* Element or attribute name associated with serialized JSON/XML data for a temp-table column.
* @param value
* Data value read from JSON content. Set to {@code null} to represent unknown value.
* @param index
* The offset value in case of an extent field. -1 to be ignored.
*
* @throws PersistenceException
* if there is an error storing data in the temp-table.
*/
protected void setField(TempTableSchema.Column column, String name, String value, int index)
throws PersistenceException
{
// handle special fields first
if (column == null)
{
switch (name)
{
case "prods:id":
if (!readingBefore)
{
// save the original prod:id in [peerMapping]
peerMapping.put(value, ctx.proxy.rowID());
}
else
{
// use the saved [prod-id] and link the before and after-images
rowid peer = peerMapping.remove(value);
if (peer != null)
{
BufferImpl after = ((BufferImpl) ctx.proxy).afterBufferNative();
if (after != null)
{
after.findByRowID(peer, LockType.EXCLUSIVE);
((TemporaryBuffer) after.buffer()).peerRowid(ctx.proxy.rowID());
((TemporaryBuffer) ((BufferImpl) ctx.proxy).buffer()).peerRowid(after.rowID());
after.release();
}
else
{
throw new PersistenceException("No default buffer of the after-image table");
}
}
// else: probably deleted? information is not yet available.
}
return;
case "prods:rowState":
int state = value.equalsIgnoreCase(STR_ROW_CREATED) ? Buffer.ROW_CREATED :
value.equalsIgnoreCase(STR_ROW_DELETED) ? Buffer.ROW_DELETED :
value.equalsIgnoreCase(STR_ROW_MODIFIED) ? Buffer.ROW_MODIFIED :
Buffer.ROW_UNMODIFIED;
((TemporaryBuffer) ((BufferImpl) ctx.proxy).buffer()).rowState(state);
if (state == Buffer.ROW_CREATED)
{
// a bit clumsy: we need to create the before image for this kind of record
BufferImpl before = ((BufferImpl) ctx.proxy).beforeBufferNative();
if (before != null)
{
before.setUpBeforeBuffer(true);
before.create();
((TemporaryBuffer) before.buffer()).rowState(Buffer.ROW_CREATED);
((TemporaryBuffer) before.buffer()).peerRowid(ctx.proxy.rowID());
((TemporaryBuffer) ((BufferImpl) ctx.proxy).buffer()).peerRowid(before.rowID());
before.setUpBeforeBuffer(false);
before.release();
}
}
return;
case Buffer.__ERROR_FLAG__:
UnimplementedFeature.missing("Set __ERROR_FLAG__ in READ-JSON/READ-XML");
return;
case Buffer.__ORIGIN_ROWID__:
UnimplementedFeature.missing("Set __ORIGIN_ROWID__ in READ-JSON/READ-XML");
return;
case Buffer.__DATA_SOURCE_ROWID__:
return; // just skip it, this hidden field is not serialized
case Buffer.__ERROR_STRING__:
UnimplementedFeature.missing("Set __ERROR_STRING__ in READ-JSON/READ-XML");
return;
case Buffer.__AFTER_ROWID__:
UnimplementedFeature.missing("Set __AFTER_ROWID__ in READ-JSON/READ-XML");
return;
}
// don't know how to handle this special column ?
throw new PersistenceException("");
}
try
{
BaseDataType datum = column.getType().getDeclaredConstructor().newInstance();
if (value == null)
{
datum.setUnknown();
}
else
{
// if the column is binary, then the element must be base64
if (column.getType() == blob.class)
{
byte[] bytes = Base64.base64ToByteArray(value);
((blob) datum).assign(bytes);
}
else
{
Stream.assignDatum(datum, value, true);
}
}
Object[] args = index != -1 ? new Object[] { index, datum } : new Object[] { datum };
column.getSetter().invoke(ctx.proxy, args);
}
catch (Exception exc)
{
throw new PersistenceException(exc);
}
}
/**
* Get schema information for the column associated with the given name.
*
* @param name
* Element or attribute name associated with serialized JSON/XML data for a temp-table column.
*
* @return Column schema information.
*
* @throws ErrorConditionException
* if {@code name} is not associated with a column.
*/
protected TempTableSchema.Column getColumn(String name)
{
switch (name)
{
case "prods:id":
case "prods:rowState":
case Buffer.__ERROR_FLAG__:
case Buffer.__ORIGIN_ROWID__:
case Buffer.__DATA_SOURCE_ROWID__:
case Buffer.__ERROR_STRING__:
case Buffer.__AFTER_ROWID__:
return null;
}
return ctx.schema.getColumn(name);
}
/**
* Process record row state: assign it to the record stored in the {@code recBuffer}.
*
* @param recBuffer
* The record buffer which will receive the provided row state.
* @param value
* The row state value.
*/
protected void processRowState(RecordBuffer recBuffer, String value)
{
int state = value.equalsIgnoreCase(STR_ROW_CREATED) ? Buffer.ROW_CREATED :
value.equalsIgnoreCase(STR_ROW_DELETED) ? Buffer.ROW_DELETED :
value.equalsIgnoreCase(STR_ROW_MODIFIED) ? Buffer.ROW_MODIFIED :
Buffer.ROW_UNMODIFIED;
((TemporaryBuffer) recBuffer).rowState(state);
if (state == Buffer.ROW_CREATED)
{
// a bit clumsy: we need to create the before image for this kind of record
BufferImpl before = ((BufferImpl) ctx.proxy).beforeBufferNative();
if (before != null)
{
before.setUpBeforeBuffer(true);
before.create();
((TemporaryBuffer) before.buffer()).rowState(Buffer.ROW_CREATED);
((TemporaryBuffer) before.buffer()).peerRowid(ctx.proxy.rowID());
((TemporaryBuffer) recBuffer).peerRowid(before.rowID());
before.setUpBeforeBuffer(false);
before.release();
}
}
}
/**
* Process record id: creates the link between the before and after-image based on the rowid value.
*
* @param recBuffer
* The record buffer containing the before image.
* @param value
* id value
*/
protected void processId(RecordBuffer recBuffer, String value)
throws PersistenceException
{
if (!readingBefore)
{
// save the original prod:id in [peerMapping]
peerMapping.put(value, ctx.proxy.rowID());
}
else
{
// use the saved [prod-id] and link the before and after-images
rowid peer = peerMapping.remove(value);
if (peer != null)
{
BufferImpl after = ((BufferImpl) ctx.proxy).afterBufferNative();
if (after != null)
{
after.findByRowID(peer, LockType.EXCLUSIVE);
((TemporaryBuffer) after.buffer()).peerRowid(ctx.proxy.rowID());
((TemporaryBuffer) recBuffer).peerRowid(after.rowID());
after.release();
}
else
{
throw new PersistenceException("No default buffer of the after-image table");
}
}
// else: probably deleted? information is not yet available.
}
}
/**
* Push import context data.
*/
protected void pushContext()
{
savedCtx.push(ctx);
this.ctx = new ImportContext(ctx);
}
/**
* Pop import context data.
*/
protected void popContext()
{
ctx = savedCtx.pop();
}
/**
* Peek the top of the import context data stack.
*
* @return the top of the import context data stack.
*/
protected ImportContext peekContext()
{
return savedCtx.peek();
}
/**
* Final setup of the inferred dataset.
*
* @param name
* The dataset name.
*
* @throws PersistenceException
* on error
*/
protected void setupDataset(String name) throws PersistenceException
{
Buffer[] buffers = new Buffer[tables.size()];
int nb = 0;
for (AbstractTempTable att: tables.values())
{
buffers[nb++] = att.defaultBufferHandleNative();
}
ds.setBuffers(buffers);
try
{
for (BufferImpl buffer: ds.getBuffers())
{
buffer.buffer().flush();
}
}
catch (ValidationException e)
{
throw new PersistenceException(e);
}
for (int nr = 0; nr < inferredRelations.size(); nr++)
{
Pair<String, String> p = inferredRelations.get(nr);
handle rel = ds.addParentIdRelation(tables.get(p.getLeft()).defaultBufferHandle(),
tables.get(p.getRight()).defaultBufferHandle());
// The relation mode is SELECTION, by default.
rel.unwrapDataRelation().setReposition(false);
}
if (name != null)
{
ds.name(name);
}
else
{
ds.name("NewDataSet");
ds.setSerializeHidden(true);
}
}
/**
* Import context holder
*/
protected static class ImportContext
{
/** Current temp-table, if one is being processed. */
public AbstractTempTable tt = null;
/** The future name of the read temp-table. */
public String ttName = null;
/** Temp-table schema information */
public TempTableSchema schema;
/** Buffer proxy */
public Buffer proxy;
/** Helper which tracks current index values for extent fields */
public ExtentTracker extentTracker;
/**
* Values of the fields to be populated from the parent table.
* (for FOREIGN-KEY-HIDDEN support)
*/
public Map<String, Object> parentValues = new HashMap<>();
/**
* Default c'tor
*/
public ImportContext()
{
super();
}
/**
* Copy c'tor
*
* @param ctx
* The instance to be cloned.
*/
public ImportContext(ImportContext ctx)
{
super();
this.tt = ctx.tt;
this.ttName = ctx.ttName;
this.schema = ctx.schema;
this.proxy = ctx.proxy;
this.extentTracker = ctx.extentTracker;
this.parentValues = ctx.parentValues;
}
/**
* Set table buffer.
*
* @param buffer
* The table buffer
* @param json
* Flag indicating if this is for JSON (when {@code true}) or XML (when {@code false}).
*/
public void setBuffer(RecordBuffer buffer, boolean json)
{
this.tt = buffer.getParentTable();
this.schema = new TempTableSchema(buffer, json);
this.proxy = (Buffer) buffer.getDMOProxy();
this.extentTracker = new ExtentTracker(schema, json);
}
}
/**
* Functional interface for adding imposed field to temp-table.
*/
@FunctionalInterface
protected static interface FieldMaker
{
/**
* Add field to the temp-table.
*
* @param ttb
* The temp-table builder.
* @param name
* The name of the field to be created.
* @param val
* The field value used to infer the field type.
*
* @throws PersistenceException
* on invalid value
*/
public void addField(TempTableBuilder ttb, String name, Object val)
throws PersistenceException;
}
}