# Importer.java

    /*
    ** Module   : Importer.java
    ** Abstract : Base class for readind JSON/XML content into a dataset/temp-table.
    **
    ** Copyright (c) 2023-2024, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
    ** 001 IAS 20230524 Created initial version.
    ** 002 CA  20230724 Further reduce context-local usage.
    ** 003 DDF 20240517 Data relations should be SELECTION by default.
    ** 004 CA  20240523 Fixed DATASET:WRITE-/READ-XML when before tables or (in)active relations are involved.
    ** 005 TJD 20240123 Java 17 compatibility updates
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist.serial;

    import static com.goldencode.p2j.persist.serial.Util.STR_ROW_CREATED;
    import static com.goldencode.p2j.persist.serial.Util.STR_ROW_DELETED;
    import static com.goldencode.p2j.persist.serial.Util.STR_ROW_MODIFIED;
    import java.lang.reflect.*;
    import java.util.*;
    import java.util.function.*;
    import org.apache.commons.lang3.tuple.*;
    import com.goldencode.p2j.convert.*;
    import com.goldencode.p2j.directory.Base64;
    import com.goldencode.p2j.persist.*;
    import com.goldencode.p2j.persist.Record;
    import com.goldencode.p2j.persist.lock.*;
    import com.goldencode.p2j.persist.serial.Importer.ImportContext;
    import com.goldencode.p2j.persist.serial.JsonImport.*;
    import com.goldencode.p2j.persist.serial.SerializeOptions.*;
    import com.goldencode.p2j.util.*;
    import com.goldencode.p2j.util.logging.*;

    /**
     * Base class for readind JSON/XML content into a dataset/temp-table.
     */
    public abstract class Importer
    {
       /** Logger. */
       private static final CentralLogger LOG = CentralLogger.get(Importer.class);

       /** Import context */
       protected ImportContext ctx = new ImportContext();

       /** Import context */
       protected final Stack<ImportContext> savedCtx = new Stack<>();

       /** Data source which normalizes access of various media to an input stream. */
       protected final SourceData source;

       /** Read mode to determine how records are stored and how non-unique records are handled */
       protected final Read readMode;

       /** Current dataset, if one is being processed. */
       protected DataSet ds = null;

       /** Child fields for NESTED relations with FOREIGN-KEY-HIDDEN = true per child table name*/
       protected Map<String, Pair<String[], String[]>> childFieldsByChild = new HashMap<>();

       /** {@code true} when reading a before-table image. */
       protected boolean readingBefore = false;

       /** Associates the {@code prod:id} to their record's {@code rowid}.*/
       protected Map<String, rowid> peerMapping = null;

       /** Helper for setting a {@code Record} into a buffer, avoiding direct access to private method. */
       protected BiConsumer<RecordBuffer, Record> loader = null;

       /**
        * The set of temp-table builders for a dataset. They are mapped by their names.
        */
       protected final Map<String, AbstractTempTable> tables = new LinkedHashMap<>();

       /** Mapping of tables with before-table names. */
       protected final Map<String, String> b4tables = new HashMap<>();

       /** Inferred DATA-RELATIONs data */
       protected final List<Pair<String, String>> inferredRelations = new ArrayList<>();

       /** The {@link BufferManager} instance. */
       protected final BufferManager bufMan = BufferManager.get();

       /**
        * C'tor
        *
        * @param   source
        *          XML data source which normalizes access of various media to an input stream.
        * @param   readMode
        *          Read mode to determine how records are stored and how non-unique records are
        *          handled. Not currently honored, except temp-table records are deleted if mode
        *          is EMPTY.
        */
       public Importer(SourceData source, String readMode)
       {
          super();
          this.source = source;
          this.readMode = (readMode == null) ? Read.MERGE : Read.valueOf(readMode.toUpperCase());
       }

       /**
        * Get an inferred type of the value.
        *
        * @param   val
        *          The value to be analyzed.
        *
        * @return  an inferred type of the value.
        *
        * @throws  PersistenceException
        *          on invalid value type
        */
       protected abstract ParmType typeOf(Object val)
       throws PersistenceException;

       /**
        * Collect relations with FOREIGN-KEY-HIDDEN = <code>true></code>
        *
        * @param   ds
        *          DATASET to be analyzed
        */
       protected void processHiddenFk(DataSet ds)
       {
          ds.getRelations().filter(r -> r.getForeignKeyHidden().booleanValue()).
          forEach(r -> {
                childFieldsByChild.put(r.getChildTable(),
                         Pair.of(r.getRelationFields(true), r.getRelationFields(false)));
          });
       }

       /**
        * Initialize values of the fields to be taken from the parent (for FOREIGN-KEY-HIDDEN support).
        *
        * @param   name
        *          The name of the child table.
        *
        * @return  <code>true</code> on success.
        */
       protected boolean setParentValues(String name)
       {
          ctx.parentValues = new HashMap<>();
          if (ds.getBuffers().isEmpty()) // empty dynamic dataset
          {
             String link = savedCtx.peek().ttName;
             if (link != null)
             {
                ctx.parentValues.putIfAbsent(link + "_id",
                         ((BufferImpl)savedCtx.peek().tt.defaultBufferHandleNative()).recordID());
             }
             return true;
          }
          Pair<String[], String[]> rfields = childFieldsByChild.get(name);
          if (rfields != null)
          {
             String[] pfields = rfields.getLeft();
             String[] cfields = rfields.getRight();
             for(int nf = 0; nf < pfields.length; nf++)
             {
                TempTableSchema.Column column = ctx.schema.getColumn(pfields[nf]);
                if (column == null)
                {
                   // log "impossible" situation?
                   return false;
                }
                try
                {
                   ctx.parentValues.put(cfields[nf], column.getGetter().invoke(ctx.proxy));
                }
                catch (IllegalAccessException | InvocationTargetException e)
                {
                   LOG.severe("", e);
                   return false;
                }
             }
          }
          return true;
       }

       /**
        * Populate fields from the parent table (for FOREIGN-KEY-HIDDEN support).
        *
        * @throws  PersistenceException
        *          on database error.
        */
       protected void fillFromParent()
       throws PersistenceException
       {
          TempTableSchema.Column column;
          for (Map.Entry<String, Object> e: ctx.parentValues.entrySet())
          {
             String fname = e.getKey();
             Object val = e.getValue();
             column = getColumn(fname);
             setField(column, fname, val == null ? null : val.toString(), false);
          }
       }

       /**
        * Persist the read values as a new record.
        *
        * @param   readValues
        *          The map containing the values of the fields mapped by their name.
        * @param   addLinkToParent
        *          Flag indicating that a link to the parent must be added.
        *
        * @return  <code>true</code> on success.
        *
        * @throws  PersistenceException
        *          in the event of a persistebnce error.
        */
       protected boolean persist(FieldMaker maker, Map<String, Object> readValues, boolean addLinkToParent)
       throws PersistenceException
       {
          if (!ctx.tt._prepared())
          {
             for (Map.Entry<String, Object> m: readValues.entrySet())
             {
                maker.addField((TempTableBuilder)ctx.tt, m.getKey(), m.getValue());
             }
             if (addLinkToParent)
             {
                String link = savedCtx.peek().ttName;
                if (link != null)
                {
                   recid rid = ((BufferImpl)savedCtx.peek().tt.defaultBufferHandleNative()).recordID();
                   String linkId = link + "_id";
                   ((TempTableBuilder)ctx.tt).addNewField(new character(linkId),
                            new character(ParmType.RECID.toString()),
                            null, null, null, null, null, null, null, null,
                            true, false);
                   readValues.putIfAbsent(linkId, rid);
                }
             }
             if (!ctx.tt.tempTablePrepare(ctx.ttName).booleanValue())
             {
                return false;
             }
          }
          tables.put(ctx.ttName, ctx.tt);
          BufferImpl buffer = (BufferImpl) ctx.tt.defaultBufferHandleNative();
          ctx.setBuffer(buffer.buffer(), false);
          return populateRecord(readValues);
       }

       /**
        * Populate record with values from JSON/XML.
        *
        * @param   readValues
        *          The values read from JSON/XML mapped by their field name.
        *
        * @return  <code>true</code> on success.
        *
        * @throws  PersistenceException
        *          in the event of a persistebnce error.
        */
       protected boolean populateRecord(Map<String, Object> readValues)
       throws PersistenceException
       {
          boolean rc = true;
          ctx.proxy.create();
          RecordBuffer.startBatch(bufMan, true);
          try
          {
             for (Map.Entry<String, Object> m: readValues.entrySet())
             {
                String name = m.getKey();
                Object val = m.getValue();
                TempTableSchema.Column column = getColumn(name);
                if (column == null)
                {
                   continue;
                }
                if (val instanceof List)
                {
                   List<Object> list = (List<Object>) val;
                   for (Object o : list)
                   {
                      setField(column, name, o == null ? null : o.toString(), true);
                   }
                }
                else
                {
                   setField(column, name, val == null ? null : val.toString(), false);
                }
             }
          }
          finally
          {
             ctx.extentTracker.reset();
             ErrorManager.ErrorHelper eh = ErrorManager.getErrorHelper();
             boolean wm = eh.isWarningMode();
             if (!wm)
             {
                eh.setWarningMode(true);
             }
             int validationErrors = RecordBuffer.endBatch(bufMan, false);
             if (!wm)
             {
                eh.setWarningMode(false);
             }
             if (ErrorManager.isPendingError() || validationErrors != 0)
             {
                ((BufferImpl) ctx.proxy).drop();
                ErrorManager.recordOrShowError(15366, true, ctx.schema.getName(), "");
                // Unable to update indexes for table '<name>'
                rc = false;
             }
          }
          readValues.clear();
          return rc;
       }

       /**
        * Set the value of a single field in the temp-table record currently being read from the JSON/XML
        * content.
        *
        * @param   column
        *          Schema information for the associated column.
        * @param   name
        *          Element or attribute name associated with serialized JSON/XML data for a temp-table
        *          column.
        * @param   value
        *          Data value read from JSON/XML content. Set to {@code null} to represent unknown value.
        *
        * @throws  PersistenceException
        *          if there is an error storing data in the temp-table.
        */
       protected void setField(TempTableSchema.Column column, String name, String value)
       throws PersistenceException
       {
          Integer index = ctx.extentTracker.getNext(name);
          setField(column, name, value, (index != null) ? index : -1);
       }

       /**
        * Set the value of a single field in the temp-table record currently being read from the JSON/XML content.
        *
        * @param   column
        *          Schema information for the associated column.
        * @param   name
        *          Element or attribute name associated with serialized JSON/XML data for a temp-table column.
        * @param   value
        *          Data value read from JSON content. Set to {@code null} to represent unknown value.
        * @param   extent
        *          {@code true} to indicate an extent field; {@code false} if scalar.
        *
        * @throws  PersistenceException
        *          if there is an error storing data in the temp-table.
        */
       protected void setField(TempTableSchema.Column column, String name, String value, boolean extent)
       throws PersistenceException
       {
          setField(column, name, value, extent ? ctx.extentTracker.getNext(name) : -1);
       }

       /**
        * Set the value of a single field in the temp-table record currently being read from the JSON/XML content.
        *
        * @param   column
        *          Schema information for the associated column.
        * @param   name
        *          Element or attribute name associated with serialized JSON/XML data for a temp-table column.
        * @param   value
        *          Data value read from JSON content. Set to {@code null} to represent unknown value.
        * @param   index
        *          The offset value in case of an extent field. -1 to be ignored.
        *
        * @throws  PersistenceException
        *          if there is an error storing data in the temp-table.
        */
       protected void setField(TempTableSchema.Column column, String name, String value, int index)
       throws PersistenceException
       {
          // handle special fields first
          if (column == null)
          {
             switch (name)
             {
                case "prods:id":
                   if (!readingBefore)
                   {
                      // save the original prod:id in [peerMapping]
                      peerMapping.put(value, ctx.proxy.rowID());
                   }
                   else
                   {
                      // use the saved [prod-id] and link the before and after-images
                      rowid peer = peerMapping.remove(value);
                      if (peer != null)
                      {
                         BufferImpl after = ((BufferImpl) ctx.proxy).afterBufferNative();
                         if (after != null)
                         {
                            after.findByRowID(peer, LockType.EXCLUSIVE);
                            ((TemporaryBuffer) after.buffer()).peerRowid(ctx.proxy.rowID());
                            ((TemporaryBuffer) ((BufferImpl) ctx.proxy).buffer()).peerRowid(after.rowID());
                            after.release();
                         }
                         else
                         {
                            throw new PersistenceException("No default buffer of the after-image table");
                         }
                      }
                      // else: probably deleted? information is not yet available.
                   }
                   return;

                case "prods:rowState":
                   int state = value.equalsIgnoreCase(STR_ROW_CREATED) ? Buffer.ROW_CREATED :
                               value.equalsIgnoreCase(STR_ROW_DELETED) ? Buffer.ROW_DELETED :
                               value.equalsIgnoreCase(STR_ROW_MODIFIED) ? Buffer.ROW_MODIFIED :
                               Buffer.ROW_UNMODIFIED;
                   ((TemporaryBuffer) ((BufferImpl) ctx.proxy).buffer()).rowState(state);
                   if (state == Buffer.ROW_CREATED)
                   {
                      // a bit clumsy: we need to create the before image for this kind of record
                      BufferImpl before = ((BufferImpl) ctx.proxy).beforeBufferNative();
                      if (before != null)
                      {
                         before.setUpBeforeBuffer(true);
                         before.create();
                         ((TemporaryBuffer) before.buffer()).rowState(Buffer.ROW_CREATED);
                         ((TemporaryBuffer) before.buffer()).peerRowid(ctx.proxy.rowID());
                         ((TemporaryBuffer) ((BufferImpl) ctx.proxy).buffer()).peerRowid(before.rowID());
                         before.setUpBeforeBuffer(false);
                         before.release();
                      }
                   }
                   return;

                case Buffer.__ERROR_FLAG__:
                   UnimplementedFeature.missing("Set __ERROR_FLAG__ in READ-JSON/READ-XML");
                   return;

                case Buffer.__ORIGIN_ROWID__:
                   UnimplementedFeature.missing("Set __ORIGIN_ROWID__ in READ-JSON/READ-XML");
                   return;

                case Buffer.__DATA_SOURCE_ROWID__:
                   return; // just skip it, this hidden field is not serialized

                case Buffer.__ERROR_STRING__:
                   UnimplementedFeature.missing("Set __ERROR_STRING__ in READ-JSON/READ-XML");
                   return;

                case Buffer.__AFTER_ROWID__:
                   UnimplementedFeature.missing("Set __AFTER_ROWID__ in READ-JSON/READ-XML");
                   return;
             }

             // don't know how to handle this special column ?
             throw new PersistenceException("");
          }

          try
          {
             BaseDataType datum = column.getType().getDeclaredConstructor().newInstance();
             if (value == null)
             {
                datum.setUnknown();
             }
             else
             {
                // if the column is binary, then the element must be base64
                if (column.getType() == blob.class)
                {
                   byte[] bytes = Base64.base64ToByteArray(value);
                   ((blob) datum).assign(bytes);
                }
                else
                {
                   Stream.assignDatum(datum, value, true);
                }
             }

             Object[] args = index != -1 ? new Object[] { index, datum } : new Object[] { datum };
             column.getSetter().invoke(ctx.proxy, args);
          }
          catch (Exception exc)
          {
             throw new PersistenceException(exc);
          }
       }

       /**
        * Get schema information for the column associated with the given name.
        *
        * @param   name
        *          Element or attribute name associated with serialized JSON/XML data for a temp-table column.
        *
        * @return  Column schema information.
        *
        * @throws  ErrorConditionException
        *          if {@code name} is not associated with a column.
        */
       protected TempTableSchema.Column getColumn(String name)
       {
          switch (name)
          {
             case "prods:id":
             case "prods:rowState":
             case Buffer.__ERROR_FLAG__:
             case Buffer.__ORIGIN_ROWID__:
             case Buffer.__DATA_SOURCE_ROWID__:
             case Buffer.__ERROR_STRING__:
             case Buffer.__AFTER_ROWID__:
                return null;
          }

          return ctx.schema.getColumn(name);
       }

       /**
        * Process record row state: assign it to the record stored in the {@code recBuffer}.
        *
        * @param   recBuffer
        *          The record buffer which will receive the provided row state.
        * @param   value
        *          The row state value.
        */
       protected void processRowState(RecordBuffer recBuffer, String value)
       {
          int state = value.equalsIgnoreCase(STR_ROW_CREATED) ? Buffer.ROW_CREATED :
                      value.equalsIgnoreCase(STR_ROW_DELETED) ? Buffer.ROW_DELETED :
                      value.equalsIgnoreCase(STR_ROW_MODIFIED) ? Buffer.ROW_MODIFIED :
                            Buffer.ROW_UNMODIFIED;
          ((TemporaryBuffer) recBuffer).rowState(state);
          if (state == Buffer.ROW_CREATED)
          {
             // a bit clumsy: we need to create the before image for this kind of record
             BufferImpl before = ((BufferImpl) ctx.proxy).beforeBufferNative();
             if (before != null)
             {
                before.setUpBeforeBuffer(true);
                before.create();
                ((TemporaryBuffer) before.buffer()).rowState(Buffer.ROW_CREATED);
                ((TemporaryBuffer) before.buffer()).peerRowid(ctx.proxy.rowID());
                ((TemporaryBuffer) recBuffer).peerRowid(before.rowID());
                before.setUpBeforeBuffer(false);
                before.release();
             }
          }
       }

       /**
        * Process record id: creates the link between the before and after-image based on the rowid value.
        *
        * @param   recBuffer
        *          The record buffer containing the before image.
        * @param   value
        *          id value
        */
       protected void processId(RecordBuffer recBuffer, String value)
       throws PersistenceException
       {
          if (!readingBefore)
          {
             // save the original prod:id in [peerMapping]
             peerMapping.put(value, ctx.proxy.rowID());
          }
          else
          {
             // use the saved [prod-id] and link the before and after-images
             rowid peer = peerMapping.remove(value);
             if (peer != null)
             {
                BufferImpl after = ((BufferImpl) ctx.proxy).afterBufferNative();
                if (after != null)
                {
                   after.findByRowID(peer, LockType.EXCLUSIVE);
                   ((TemporaryBuffer) after.buffer()).peerRowid(ctx.proxy.rowID());
                   ((TemporaryBuffer) recBuffer).peerRowid(after.rowID());
                   after.release();
                }
                else
                {
                   throw new PersistenceException("No default buffer of the after-image table");
                }
             }
             // else: probably deleted? information is not yet available.
          }
       }

       /**
        * Push import context data.
        */
       protected void pushContext()
       {
          savedCtx.push(ctx);
          this.ctx = new ImportContext(ctx);
       }

       /**
        * Pop import context data.
        */
       protected void popContext()
       {
          ctx = savedCtx.pop();
       }

       /**
        * Peek the top of the import context data stack.
        *
        * @return  the top of the import context data stack.
        */
       protected ImportContext peekContext()
       {
          return savedCtx.peek();
       }

       /**
        * Final setup of the inferred dataset.
        *
        * @param   name
        *          The dataset name.
        *
        * @throws  PersistenceException
        *          on error
        */
       protected void setupDataset(String name) throws PersistenceException
       {
          Buffer[] buffers = new Buffer[tables.size()];
          int nb = 0;
          for (AbstractTempTable att: tables.values())
          {
             buffers[nb++] = att.defaultBufferHandleNative();
          }
          ds.setBuffers(buffers);
          try
          {
             for (BufferImpl buffer: ds.getBuffers())
             {
                buffer.buffer().flush();
             }
          }
          catch (ValidationException e)
          {
             throw new PersistenceException(e);
          }
          for (int nr = 0; nr < inferredRelations.size(); nr++)
          {
             Pair<String, String> p = inferredRelations.get(nr);
             handle rel = ds.addParentIdRelation(tables.get(p.getLeft()).defaultBufferHandle(),
                                                 tables.get(p.getRight()).defaultBufferHandle());
             // The relation mode is SELECTION, by default.
             rel.unwrapDataRelation().setReposition(false);
          }
          if (name != null)
          {
             ds.name(name);
          }
          else
          {
             ds.name("NewDataSet");
             ds.setSerializeHidden(true);
          }
       }

       /**
        * Import context holder
        */
       protected static class ImportContext
       {
          /** Current temp-table, if one is being processed. */
          public AbstractTempTable tt = null;

          /** The future name of the read temp-table. */
          public String ttName = null;

          /** Temp-table schema information */
          public TempTableSchema schema;

          /** Buffer proxy */
          public Buffer proxy;

          /** Helper which tracks current index values for extent fields */
          public ExtentTracker extentTracker;

          /**
           * Values of the fields to be populated from the parent table.
           * (for FOREIGN-KEY-HIDDEN support)
           */
          public Map<String, Object> parentValues = new HashMap<>();

          /**
           * Default c'tor
           */
          public ImportContext()
          {
             super();
          }

          /**
           * Copy c'tor
           *
           * @param   ctx
           *          The instance to be cloned.
           */
          public ImportContext(ImportContext ctx)
          {
             super();
             this.tt = ctx.tt;
             this.ttName = ctx.ttName;
             this.schema = ctx.schema;
             this.proxy = ctx.proxy;
             this.extentTracker = ctx.extentTracker;
             this.parentValues = ctx.parentValues;
          }

          /**
           * Set table buffer.
           *
           * @param   buffer
           *          The table buffer
           * @param   json
           *          Flag indicating if this is for JSON (when {@code true}) or XML (when {@code false}).
           */
          public void setBuffer(RecordBuffer buffer, boolean json)
          {
             this.tt = buffer.getParentTable();
             this.schema = new TempTableSchema(buffer, json);
             this.proxy = (Buffer) buffer.getDMOProxy();
             this.extentTracker = new ExtentTracker(schema, json);
          }
       }

       /**
        * Functional interface for adding imposed field to temp-table.
        */
       @FunctionalInterface
       protected static interface FieldMaker
       {
          /**
           * Add field to the temp-table.
           *
           * @param   ttb
           *          The temp-table builder.
           * @param   name
           *          The name of the field to be created.
           * @param   val
           *          The field value used to infer the field type.
           *
           * @throws  PersistenceException
           *          on invalid value
           */
          public void addField(TempTableBuilder ttb, String name, Object val)
          throws PersistenceException;
       }
    }