FastCopyHelper.java
/*
** Module : FastCopyHelper.java
** Abstract : The core of fast-copy technique implementation
**
** Copyright (c) 2020-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
** 001 AIL 20201201 First revision.
** 002 AIL 20201203 Added append mode support.
** 20201215 Temporarily avoided fast-copy for loose-copy-mode.
** 20201216 Read support for loose-copy-mode.
** OM 20210223 Invalidate index-based cache in the event of low-level SQL operations. Replaced iterated
** string concatenation with append chaining.
** CA 20210506 Unique constraints sub-selects must use a different alias for the table.
** ECF 20210506 Use RecordBuffer.getDmoInfo() getter instead of direct field access to prevent NPE in
** proxy case.
** CA 20210526 Fixed computeUniqueConstraintsClauses - the dst/src lists were reversed.
** CA 20210928 Fixed getMapping - the pkMappingTableName parameter must be used.
** OM 20211020 Added _DATASOURCE_ROWID property.
** OM 20220727 FieldId and PropertyId are different for denormalized extent fields.
** DDF 20220823 Replaced .nextvalue with next value for.
** ECF 20221006 Minor optimization.
** OM 20220913 Enhanced dialect API for sorting nulls.
** OM 20221012 Fixed regression in previous commit.
** RAA 20230109 Changed inline statement(s) to prepared statement(s).
** SVL 20230110 P2JIndex.components() provides direct access to the components array in order to improve
** performance.
** SVL 20230113 Improved performance by replacing some "for-each" loops with indexed "for" loops.
** RAA 20230314 Multiplex id is now an argument rather than being hard-coded. This takes effect in
** copyTable and safeTableCopy.
** DDF 20230306 Made a change so that temp-tables don't use computed columns anymore (refs: #7108).
** 003 DDF 20230523 Use wildcard when possible instead of using all columns into the copy statement.
** DDF 20230524 Fix uid usage and replace ALLOW_WILDCARD with dialect specific method.
** DDF 20230525 Add flag that enables uid usage. At the moment, usage of uid for intermediate table
** increases the size of the psCache.
** 004 DDF 20230606 Added a new copy method that doesn't use mapping.
** DDF 20230612 Fix missing records when doing a safe copy without mapping.
** SR 20230721 Let append be false when copying tables if the destination is empty.
** AL2 20230826 Reset loose-copy flag if the property matching is right (source matches destination).
** SR 20230726 Updated updatePeerRecords to use preparedStatements instead of plain SQL statements.
** RAA 20230807 Created FastCopyBuilder and a cache for instances of FastCopyHelper.
** Refactored class so it doesn't contain static methods anymore (apart from cache usage).
** RAA 20230823 Merged master and before processes and added a bundle that stores the SQLs which will be
** executed.
** RAA 20230828 Removed SQL storing variables. Got rid of the propagated "append" flag.
** RAA 20230830 Removed tryExecuteCopy and changed so that before copy can access safeTableCopy.
** RAA 20230904 Renamed finishCopy into executeCopyImpl, which is now called from executeCopy.
** AL2 20230905 Fixed noMapping flag to be part of the helper, not the context.
** Removed replaceMode from fast-copy as it is not supported; removed CopyMeta.
** 005 AL2 20230918 Fixed regression when safeTableCopyNoMapping was not honoring unique constraints.
** AL2 20230919 Fixed regression where updatePeerRecords may prematurely stop.
** 006 AL2 20231012 Replaced getCurrentPrimaryKey with nextPrimaryKey.
** 007 AL2 20231101 Created the intermediary tables only once per session and reuse between copies.
** 008 AL2 20231103 Made the dropMappings use the session instead of the persistence.
** 009 AL2 20231103 Replaced IF NOT EXISTS with IF EXISTS from drop.
** 010 AL2 20240512 Allow H2 to provide the PK instead of a separate sequence to honor reclaiming.
** AL2 20240513 Fast copy with non-expanded extent fields should also create an in-memory PK mapping.
** AL2 20240514 Removed FastCopyBundlePKElement.
** 011 SP 20240523 Fixed a for loop in safeTableCopy() when setting arguments for generatePkMappingArgs
** 012 OM 20241128 Multi-tenant runtime support: selected the proper persistence context, eventually
** based on [sharedDb] parameter.
** 013 DDF 20240205 Added the mandatory value of the index component to orderByNulls call.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.persist;
import java.sql.Savepoint;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.function.Supplier;
import com.goldencode.p2j.persist.RecordBuffer.DatumAccess;
import com.goldencode.p2j.persist.dialect.Dialect;
import com.goldencode.p2j.persist.orm.*;
import com.goldencode.p2j.util.*;
/**
* This helper is used when we need to copy records from one temp-table to another in bulk mode.
* Using statements like "insert into select from" significantly improves the time performance.
* However this technique can be used only when the source and destination tables are compatible.
* Also, this helper operates directly with the persistence layer (without using intermediate buffer
* validations and such). This means that we need to ensure at SQL level that the records are copied
* correctly.
*/
public class FastCopyHelper
{
/** Name of the multiplex ID field in temp table DMO classes */
private static final String MULTIPLEX_FIELD_NAME = TemporaryBuffer.MULTIPLEX_FIELD_NAME;
/** Name of a generic use intermediate table, used for storing the indexes of an extent */
private static final String LIST__INDEX__TABLE = "LIST__INDEX__TABLE";
/** Name of the field in a database-level pk mapping, which stores a destination pk */
private static final String DST__PK = "DST__PK";
/** Name of the field in a database-level pk mapping, which stores a source pk */
private static final String SRC__PK = "SRC__PK";
/** The name of the db level mapping used intermediary by the copy process for master tables*/
private static final String MASTER_PK_MAPPING = "MASTER__PK__MAPPING";
/** The name of the db level mapping used intermediary by the copy process for before tables*/
private static final String BEFORE_PK_MAPPING = "BEFORE__PK__MAPPING";
/** Flag used to enable usage of uid for intermediate tables */
private static final boolean ALLOW_UID = false;
/** A value used to increment the uid upon instantiation of FastCopyHelper. */
private static final AtomicLong uidCounter = new AtomicLong(0);
/** The bundle of SQLs that will be executed to perform the copy process. */
private final FastCopyBundle fastCopyBundle = new FastCopyBundle();
/** Flag which indicates that this is a simpleCopy (no need for property mapping) */
private final boolean simpleCopy;
/** Flag that indicates whether copying should be executed in APPEND mode or not. */
private final boolean appendMode;
/** Flag that indicates if this is a loose copy process. */
private final boolean looseCopyMode;
/** Variable that stores the result of the {@code canUseFastCopy} function. */
private Boolean canFastCopy;
/** Name of a generic use temporary table, used for intermediary operations. */
private final String pkMappingName;
/** Flag set when a database level mapping exists for the copy operation. */
private boolean generatesPkMapping;
/** The persistence of the source. */
private Persistence srcPersistence;
/** The meta information of the source. */
private DmoMeta srcMeta;
/** The meta information of the destination. */
private DmoMeta dstMeta;
/** The DMO interface of the source. */
private Class<? extends DataModelObject> srcDmoInterface;
/** The DMO interface of the destination. */
private Class<? extends DataModelObject> dstDmoInterface;
/** The dialect of the source. */
private Dialect srcDialect;
/** The source table name. */
private String srcTableName;
/** The destination table name. */
private String dstTableName;
/** The source index name. */
private String srcImplicitIndexName;
/** The source index name used in SQLs. */
private String srcImplicitSqlIndexName;
/** A property mapping in case this is not a simple copy. */
private Map<DatumAccess, DatumAccess> propsMap;
/** The list of properties specific to the source temp-table. */
private List<Property> srcSimpleProps;
/** The list of properties specific to the destination temp-table. */
private List<Property> dstSimpleProps;
/**
* A collection of extent fields. The key is the extent.
* The value is a list of two properties (the source property and destination property).
*/
private Map<Integer, List<Property[]>> extentFields;
/** A cache for the sort clause used when copying tables. */
private String sort;
/** A value used to generate unique names for the tables used for intermediate operations and storing. */
private long uid;
/** {@code true} if this fast-copy helper will force the creation of an intermediate PK mapping */
private boolean forceMapping;
/**
* Create the in-database mappings which are used for intermediary steps in the fast-copy process.
* Complex copy attempts implying extent tables and before tables, require a PK mapping to join with.
* This PK mapping is stored in these intermediary tables.
*
* Creating only one pair of tables per session (one for master and one for before) is an optimization.
* Creating and dropping at each fast-copy bundle will force the reparsing of the statements using the
* PK mapping tables.
*
* @param persistence
* The persistence in which the tables should be created.
*/
public static void createMappings(Persistence persistence)
{
try
{
String fields = " (" + DST__PK + " bigint, " + SRC__PK + " bigint, " + TempRecord._PEER_ROWID + " bigint)";
persistence.executeSQL(
"create local temporary table if not exists " + MASTER_PK_MAPPING + fields + " transactional",
Persistence.TEMP_CTX);
persistence.executeSQL(
"create local temporary table if not exists " + BEFORE_PK_MAPPING + fields + " transactional",
Persistence.TEMP_CTX);
}
catch (PersistenceException exc)
{
throw new IllegalStateException(exc);
}
}
/**
* Drop the in-database mappings which are used for intermediary steps in the fast-copy process.
* Use the session instead of the persistence as this is usually called in a clean-up routine where we don't have
* access to the context anymore
*
* @param session
* The persistence from which the tables should be dropped.
*
* @throws PersistenceException
* This is thrown if the persistence layer fails
*/
public static void dropMappings(Session session)
throws PersistenceException
{
Session.createSQLQuery("drop table if exists " + MASTER_PK_MAPPING + " transactional").executeUpdate(session);
Session.createSQLQuery("drop table if exists " + BEFORE_PK_MAPPING + " transactional").executeUpdate(session);
}
/**
* Constructor that is based on the context given.
*
* @param fastCopyContext
* The context that has the information required.
*/
private FastCopyHelper(FastCopyContext fastCopyContext)
{
this.simpleCopy = fastCopyContext.simpleCopy;
this.propsMap = fastCopyContext.propsMap;
this.appendMode = fastCopyContext.appendMode;
this.looseCopyMode = fastCopyContext.looseCopyMode;
this.forceMapping = fastCopyContext.forceMapping;
this.uid = uidCounter.getAndIncrement();
this.pkMappingName = fastCopyContext.isBefore ? BEFORE_PK_MAPPING : MASTER_PK_MAPPING;
initialize(fastCopyContext.srcBuf, fastCopyContext.dstBuf);
}
/**
* This method will update the peer values in the master and before tables. This is mandatory
* as long as before tables exist. Otherwise, inconsistencies may appear.
*
* @param masterCopyHelper
* The master instance that handled the master copy.
* @param masterCopyContext
* The master context that handled the master copy.
* @param beforeCopyHelper
* The before instance that handled the before copy.
* @param beforeCopyContext
* The before context that handled the before copy.
*/
static void updatePeerRecords(FastCopyHelper masterCopyHelper,
FastCopyContext masterCopyContext,
FastCopyHelper beforeCopyHelper,
FastCopyContext beforeCopyContext)
throws PersistenceException
{
// TODO: check if we can't update peer records using the database level mapping
// in order to avoid additional mapping query and sequential update statements
Map<Long, Long[]> srcToDstPks = masterCopyHelper.getMapping(masterCopyContext);
Map<Long, Long[]> srcB4ToDstB4Pks = beforeCopyHelper.getMapping(beforeCopyContext);
Iterator<Long[]> it = srcToDstPks.values().iterator();
Iterator<Long[]> itB4 = srcB4ToDstB4Pks.values().iterator();
String sql = beforeCopyHelper.getDstTableSql();
Supplier<Object[]> supp = () ->
{
Long[] result = null;
while (it.hasNext() && (result == null || result[1] == null))
{
result = it.next();
}
if (result == null || result[1] == null)
{
return null;
}
Long dstPk = result[0];
Long srcPeerId = result[1];
Long dstB4Pk = srcB4ToDstB4Pks.get(srcPeerId)[0];
return new Object[] {dstPk, dstB4Pk};
};
beforeCopyHelper.srcPersistence.executeSQLBatch(sql, Persistence.TEMP_CTX, supp);
sql = masterCopyHelper.getDstTableSql();
Supplier<Object[]> b4Supp = () ->
{
Long[] result = null;
while (itB4.hasNext() && (result == null || result[1] == null))
{
result = itB4.next();
}
if (result == null || result[1] == null)
{
return null;
}
Long dstB4Pk = result[0];
Long srcB4PeerId = result[1];
Long dstPk = srcToDstPks.get(srcB4PeerId)[0];
return new Object[] {dstB4Pk, dstPk};
};
masterCopyHelper.srcPersistence.executeSQLBatch(sql, Persistence.TEMP_CTX, b4Supp);
}
/**
* Performance improved variant, when we know the destination table can have its
* records inserted via <code>INSERT INTO ... SELECT FROM</code>, without having to do any validation on
* the records, as the destination table will accept them all.
*
* If the copy bundle is already computed, then this method will simply determine the execution of the
* SQLs. Otherwise, It will generate them and then it will execute.
*
* @param fastCopyContext
* The context that has the required information.
* @param srcRecId
* A source record id for which a destination record id should be computed at the end.
* If this is null, then null will be returned.
*
* @return The recid in the destination equivalent to the provided source recid if not {@code null}.
*/
rowid executeCopy(FastCopyContext fastCopyContext, rowid srcRecId)
throws PersistenceException
{
if (fastCopyBundle.isBundleComputed())
{
return executeCopyImpl(fastCopyContext, srcRecId);
}
boolean loose = !processProperties();
copyTable(fastCopyContext, loose);
if (extentFields != null && !extentFields.isEmpty())
{
copyExtentFields(fastCopyContext);
}
return executeCopyImpl(fastCopyContext, srcRecId);
}
/**
* This should be used to determine if this helper can be used in order to execute a fast copy
* on the provided temp-tables.
*
* @return {@code true} if this helper can be used for fast-copy.
*/
boolean canUseFastCopy()
{
if (canFastCopy != null)
{
return canFastCopy;
}
if (!DMOSignatureHelper.validTempTableBulkCopy(srcMeta.getSignature(),
dstMeta.getSignature(),
looseCopyMode))
{
canFastCopy = false;
return false;
}
if (looseCopyMode)
{
// make sure that all unique constraints are satisfied by the copied fields
// in other words, avoid fast copy if we know that there are fields in the unique constraints
// which will be set on the default value
Set<Integer> destinationProps = new HashSet<>();
if (propsMap != null)
{
for (Map.Entry<DatumAccess, DatumAccess> entry : propsMap.entrySet())
{
destinationProps.add(entry.getValue().getPropertyMeta().getId());
}
}
ArrayList<List<Property>> uniqueConstraints = dstMeta.getUniqueConstraintsAsProps();
for (int i = 0; i < uniqueConstraints.size(); i++)
{
List<Property> props = uniqueConstraints.get(i);
for (int j = 0; j < props.size(); j++)
{
Property prop = props.get(j);
if (!destinationProps.contains(prop.propId))
{
canFastCopy = false;
return false;
}
}
}
}
canFastCopy = true;
return true;
}
/**
* Particular method which clears the intermediate database-level pk mapping for the table copy.
* This will eventually drop any such table.
*
* @param ctx
* The fast-copy context in which this pk mapping clear is done.
*/
void clear(FastCopyContext ctx)
throws PersistenceException
{
if (ctx.hasMapping)
{
// TODO: replace with truncate table once FWD-H2 supports transactional for TRUNCATE TABLE
srcPersistence.executeSQL(
"delete from " + assembleUidMapping(pkMappingName) /* + transactional */,
Persistence.TEMP_CTX);
ctx.hasMapping = false;
}
}
/**
* Given the buffers used, initialize the fields that will be required in this class.
*
* @param srcBuf
* The source buffer.
* @param dstBuf
* The destination buffer.
*/
private void initialize(TemporaryBuffer srcBuf, TemporaryBuffer dstBuf)
{
if (srcBuf != null)
{
srcMeta = srcBuf.getDmoInfo();
srcPersistence = srcBuf.getPersistence();
srcDmoInterface = srcBuf.getDMOInterface();
srcDialect = srcBuf.getDialect();
srcTableName = srcBuf.getDmoInfo().getSqlTableName();
srcImplicitIndexName = srcBuf.getImplicitIndexName(srcDmoInterface);
srcImplicitSqlIndexName = srcBuf.getImplicitSqlIndexName(srcDmoInterface);
}
if (dstBuf != null)
{
dstMeta = dstBuf.getDmoInfo();
dstDmoInterface = dstBuf.getDMOInterface();
dstTableName = dstBuf.getDmoInfo().getSqlTableName();
}
}
/**
* Method used to execute the SQLs for the copy process and, if needed, set the destination rowid.
*
* @param fastCopyContext
* The context with the required information.
* @param srcRecId
* The {@link rowid} of the current record in the source buffer - if not null, it will be set in
* the destination buffer, too.
*
* @return The destination {@code rowid}.
*/
private rowid executeCopyImpl(FastCopyContext fastCopyContext, rowid srcRecId)
throws PersistenceException
{
// execute the SQL bundle
fastCopyBundle.executeSqls(fastCopyContext);
if (generatesPkMapping)
{
fastCopyContext.hasMapping = true;
}
rowid result = null;
if (srcRecId != null)
{
Map<Long, Long[]> pkMapping = getMapping(fastCopyContext);
result = new rowid(pkMapping.get(srcRecId.getValue())[0]);
}
return result;
}
/**
* The main procedure involved when copying the records from a source to a destination.
* This processes the tables and executes a fast table copy SQL meant to insert into the destination
* a record selection from the source.
* The properties given as parameters are already ordered such that the insert SQL will automatically
* match the correct fields.
*
* @param fastCopyContext
* The context that has the required information.
* @param srcSimpleProps
* A list of properties in the source table.
* @param dstSimpleProps
* A list of properties in the destination table.
* @param sort
* The sort clause which should be used when computing the insert SQL.
* @param looseCopy
* If the copy is in loose mode.
*
* @return The bundle element whose execution will do the copy.
*/
private FastCopyBundleElement copyTable(FastCopyContext fastCopyContext,
List<Property> srcSimpleProps,
List<Property> dstSimpleProps,
String sort,
boolean looseCopy)
{
// compute the columns which should be involved in the insertion in the right order
StringBuilder columnListSrc;
StringBuilder columnListDst;
List<Object> args = new ArrayList<>();
if (!srcDialect.allowSelectWildcard() ||
looseCopy ||
srcMeta.hasComputedColumns(false) ||
dstMeta.hasComputedColumns(false))
{
columnListSrc = getSrcTempColumns(dstTableName);
columnListDst = getDstTempColumns();
for (int i = 0; i < srcSimpleProps.size(); i++)
{
appendColumnName(srcMeta,
dstMeta,
srcSimpleProps.get(i),
dstSimpleProps.get(i),
columnListSrc,
columnListDst,
args);
}
}
else
{
columnListSrc = getSrcTempWildcard(dstTableName, srcTableName);
columnListDst = new StringBuilder();
}
String sqlInsert = getFastTableCopySql(srcTableName,
dstTableName,
columnListSrc.toString(),
columnListDst.toString(),
srcImplicitSqlIndexName,
sort);
Function<FastCopyContext, Object[]> generateArgs = (context) ->
{
int argSize = args.size();
Object[] arguments = new Object[argSize + 3];
arguments[0] = context.dstBuf.getMultiplexID();
arguments[1] = context.dstBuf.getMultiplexID();
for (int i = 0; i < argSize; i++)
{
arguments[i + 2] = args.get(i);
}
arguments[argSize + 2] = context.srcBuf.getMultiplexID();
return arguments;
};
return new FastCopyBundleElement(sqlInsert, generateArgs);
}
/**
* A more complex way to copy a temp-table which ensures that the unique constraints won't
* be violated after copy. This only checks that the new records are not already in the
* destination (according to any unique constraint). It does not check for duplicates in the
* source; this should be ruled out by the compatibility checker {@link #canUseFastCopy()}.
* This technique uses a pre-copy pk mapping, which means that at first we will have a
* preselection of the records from the source. Afterwards, we apply a "shadow" copy, meaning
* that we won't check the constraints anymore; we just copy the preselected records from
* the source to the destination.
* This will create a pk mapping. This won't allow post-copy pk mappings as they won't be safe.
*
* @param fastCopyContext
* The context that has the required information.
* @param srcSimpleProps
* A list of properties in the source table.
* @param dstSimpleProps
* A list of properties in the destination table.
* @param sort
* The sort clause which should be used when computing the insert SQL.
* @param pkMappingTableName
* The name of the intermediate table which will be used for pk mapping.
* @param looseCopy
* If the copy is in loose mode.
*
* @return The bundle element whose execution will do the copy.
*/
private FastCopyBundleElement safeTableCopy(FastCopyContext fastCopyContext,
List<Property> srcSimpleProps,
List<Property> dstSimpleProps,
String sort,
String pkMappingTableName,
boolean looseCopy)
{
List<String> whereClauses = computeUniqueConstraintsClauses(srcSimpleProps, dstSimpleProps);
String pkMappingSql = getPkMappingSql(srcTableName,
dstTableName,
srcImplicitSqlIndexName,
whereClauses,
sort,
pkMappingTableName);
Function<FastCopyContext, Object[]> generatePkMappingArgs = (context) ->
{
int size = whereClauses.size();
Object[] arguments = new Object[size + 2];
Integer multiplexId = context.dstBuf.getMultiplexID();
arguments[0] = multiplexId;
arguments[1] = context.srcBuf.getMultiplexID();
for (int i = 0; i < size; i++)
{
arguments[i + 2] = multiplexId;
}
return arguments;
};
fastCopyBundle.addBundleElement(new FastCopyBundleElement(pkMappingSql, generatePkMappingArgs));
// we start preparing the shadow copy sql
StringBuilder columnListSrc;
StringBuilder columnListDst;
List<Object> args = new ArrayList<>();
if (!srcDialect.allowSelectWildcard() ||
looseCopy ||
srcMeta.hasComputedColumns(false) ||
dstMeta.hasComputedColumns(false))
{
columnListSrc = getSrcMappedColumns(pkMappingTableName);
columnListDst = getDstTempColumns();
for (int i = 0; i < srcSimpleProps.size(); i++)
{
appendColumnName(srcMeta,
dstMeta,
srcSimpleProps.get(i),
dstSimpleProps.get(i),
columnListSrc,
columnListDst,
args);
}
}
else
{
columnListSrc = getSrcMappedWildcard(pkMappingTableName, srcTableName);
columnListDst = new StringBuilder();
}
// we already have a pk mapping at database-level
// using that pk mapping, copy rows from source to destination, considering that
// the pk mapping is constraint safe (the unique constraints are already checked,
// the order is correct and the _multiplex is the one desired)
String shadowCopySQL = getShadowCopySql(srcTableName,
dstTableName,
columnListSrc.toString(),
columnListDst.toString(),
sort,
pkMappingTableName);
Function<FastCopyContext, Object[]> generateShadowCopyArgs = (context) ->
{
int size = args.size();
Object[] arguments = new Object[size + 1];
arguments[0] = context.dstBuf.getMultiplexID();
for (int i = 0; i < size; i++)
{
arguments[i + 1] = args.get(i);
}
return arguments;
};
return new FastCopyBundleElement(shadowCopySQL,
generateShadowCopyArgs);
}
/**
* Handles the copy of the records from a source to a destination without using
* intermediate tables. This type of copy is done only in append mode when there
* is no need for mapping afterwards for extents/before table/destRecId. The mapping
* is replaced by "inlining" it into the where clause and using an additional
* condition for the recid to avoid an infinite loop.
*
* @param fastCopyContext
* The context that has the required information.
* @param srcSimpleProps
* A list of properties in the source table.
* @param dstSimpleProps
* A list of properties in the destination table.
* @param sort
* The sort clause which should be used when computing the insert SQL.
* @param looseCopy
* If the copy is in loose mode.
*
* @return The bundle element whose execution will do the copy.
*/
private FastCopyBundleElement safeTableCopyNoMapping(FastCopyContext fastCopyContext,
List<Property> srcSimpleProps,
List<Property> dstSimpleProps,
String sort,
boolean looseCopy)
{
StringBuilder columnListSrc;
StringBuilder columnListDst;
List<Object> args = new ArrayList<>();
if (!srcDialect.allowSelectWildcard() ||
looseCopy ||
srcMeta.hasComputedColumns(false) ||
dstMeta.hasComputedColumns(false))
{
columnListSrc = getSrcTempColumns(dstTableName);
columnListDst = getDstTempColumns();
for (int i = 0; i < srcSimpleProps.size(); i++)
{
appendColumnName(srcMeta,
dstMeta,
srcSimpleProps.get(i),
dstSimpleProps.get(i),
columnListSrc,
columnListDst,
args);
}
}
else
{
columnListSrc = getSrcTempWildcard(dstTableName, srcTableName);
columnListDst = new StringBuilder();
}
List<String> whereClauses = computeUniqueConstraintsClauses(srcSimpleProps, dstSimpleProps);
String insertSQL = getSafeTableCopySqlNoMapping(srcTableName,
dstTableName,
columnListSrc.toString(),
columnListDst.toString(),
srcImplicitSqlIndexName,
whereClauses,
sort);
Function<FastCopyContext, Object[]> generateArgs = (context) ->
{
int argsSize = args.size();
Object[] arguments = new Object[argsSize + whereClauses.size() + 3];
Integer multiplexId = context.dstBuf.getMultiplexID();
arguments[0] = multiplexId;
arguments[1] = multiplexId;
for (int i = 0; i < argsSize; i++)
{
arguments[i + 2] = args.get(i);
}
arguments[argsSize + 2] = context.srcBuf.getMultiplexID();
// arguments[argsSize + 3] = context.srcBuf.nextPrimaryKey(true);
for (int i = 0; i < whereClauses.size(); i++)
{
arguments[argsSize + 3 + i] = multiplexId;
}
return arguments;
};
return new FastCopyBundleElement(insertSQL, generateArgs);
}
/**
* Identify a list of where clauses which rule out duplicate records when copying from a source
* to a destination. A duplicate record is one which will violate destination's unique constraints.
* There are multiple such clauses, one for each unique constraint. Each clause will check the
* multiplex value and the properties which take part in the constraint.
*
* @param srcSimpleProps
* A list of properties in the source table.
* @param dstSimpleProps
* A list of properties in the destination table.
*
* @return A list of SQL where clauses which can preselect records from the source which
* won't violate the unique constraints from the destination.
*/
private List<String> computeUniqueConstraintsClauses(List<Property> srcSimpleProps,
List<Property> dstSimpleProps)
{
List<String> uniqueConstraintsClauses = new ArrayList<>();
String dstAlias = dstTableName + "___dst";
ArrayList<List<Property>> constraints = dstMeta.getUniqueConstraintsAsProps();
for (int i = 0; i < constraints.size(); i++)
{
List<Property> index = constraints.get(i);
if (index == null || index.isEmpty())
{
continue;
}
StringBuilder sb = new StringBuilder("not exists (");
sb.append("select 0 from ").append(dstTableName).append(" ").append(dstAlias).append(" where ");
sb.append(dstAlias).append(".").append(MULTIPLEX_FIELD_NAME).append(" = ?");
for (int j = 0; j < index.size(); j++)
{
Property prop = index.get(j);
String dstPropName = prop.column;
String srcPropName = getMappedPropName(dstPropName, dstSimpleProps, srcSimpleProps);
sb.append(" and ");
sb.append(srcTableName).append(".").append(srcPropName).append(" = ");
sb.append(dstAlias).append(".").append(dstPropName);
}
sb.append(")");
uniqueConstraintsClauses.add(sb.toString());
}
return uniqueConstraintsClauses;
}
/**
* Identify the name of the property in the destination which should match the provided property
* from the source. They should be on the same position in the list of properties. Note that
* the result may be the same as the input (in case of a simple copy; the DMO implementation of
* the source is the same for destination), but generally they can be different.
*
* @param propName
* The property name for which we search a correspondent.
* @param from
* The list of properties for the DMO containing the provided property name.
* @param to
* The list of properties in which we search for a correspondent.
*
* @return The name of the correspondent property or null if not found.
*/
private String getMappedPropName(String propName, List<Property> from, List<Property> to)
{
for (int i = 0; i < from.size(); i++)
{
if (from.get(i).column.equals(propName))
{
return to.get(i).column;
}
}
return null;
}
/**
* The procedure which handles the copy between extent tables (only when source and destination properties
* are both normalized). For this part to work fast (so use the "insert into select from" paradigm), we
* will need a primary key mapping. Unless one is already created, this method will create a database
* level pkMapping (which can be used further in other scenarios).
*
* @param extentFields
* The mapping between the source extent fields and destination extent fields. This mapping
* is done per extent value.
* @param pkMappingTableName
* The name of the table which will be used as a database level primary key mapping.
* @param uid
* Value used to generate unique names for intermediate tables.
*/
private void copyExtentFields(Map<Integer, List<Property[]>> extentFields,
String pkMappingTableName,
long uid)
{
// insert records in all extent tables
for (Map.Entry<Integer, List<Property[]>> extent : extentFields.entrySet())
{
// prepare the columns which will be part of the copy - in the right order
StringBuilder extColumnListSrc = new StringBuilder(DST__PK + ", " +
ReservedProperty.LIST__INDEX.column);
StringBuilder extColumnListDst = new StringBuilder(ReservedProperty.PARENT__ID.column + ", " +
ReservedProperty.LIST__INDEX.column);
// there is a possibility that there is no extent table for the source
// according to the property mapping, destination's extent field is unmapped
// but we still need to set it on a default value
// so make sure don't copy from an inexistent table (source extent table)
boolean srcExtTableExists = false;
List<Object> args = new ArrayList<>();
for (Property[] props : extent.getValue())
{
if (props[0] != null)
{
srcExtTableExists = true;
}
appendColumnName(srcMeta,
dstMeta,
props[0],
props[1],
extColumnListSrc,
extColumnListDst,
args);
}
String insertSql;
if (srcExtTableExists)
{
// insert into the destination extent table all records from source
// but with re-mapped id (based on pkMappingTableName)
insertSql = getExtentInsertSql(srcTableName + "__" + extent.getKey(),
dstTableName + "__" + extent.getKey(),
extColumnListSrc.toString(),
extColumnListDst.toString(),
pkMappingTableName);
}
else
{
// insert into the destination extent table defaults
insertSql = getSimpleExtentInsertSql(extent.getKey(),
dstTableName + "__" + extent.getKey(),
extColumnListSrc.toString(),
extColumnListDst.toString(),
pkMappingTableName,
uid);
}
Function<FastCopyContext, Object[]> generateArgs = (context) ->
{
return args.toArray();
};
fastCopyBundle.addBundleElement(new FastCopyBundleElement(insertSql, generateArgs));
}
}
/**
* This will create a database-level primary key mapping which can be used in other operations
* (through join) in order to speed up the copy process.
*
* @param fastCopyContext
* The context that has the required information.
* @param sort
* The sort clause used when the records were copied between main tables.
* @param pkMappingTableName
* The name which should be used by the table which will store the primary key mapping.
* This does not check if the pkMappingTableName already exists.
* @param addToBundle
* {@code true} if instead of immediately executing the SQLs, they can be added to the bundle
* and be executed at the end, {@code false} otherwise.
*/
private void createPkMapping(FastCopyContext fastCopyContext,
String sort,
String pkMappingTableName,
boolean addToBundle)
throws PersistenceException
{
String createMapping = getPkMappingSql(srcTableName,
dstTableName,
srcImplicitSqlIndexName,
null,
sort,
pkMappingTableName);
if (addToBundle)
{
Function<FastCopyContext, Object[]> generateArgs = (context) ->
{
Object[] arguments = new Object[]
{
context.dstBuf.getMultiplexID(),
context.srcBuf.getMultiplexID()
};
return arguments;
};
fastCopyBundle.addBundleElement(new FastCopyBundleElement(createMapping, generateArgs));
return;
}
Object[] multiplexArg = new Object[]
{
fastCopyContext.dstBuf.getMultiplexID(),
fastCopyContext.srcBuf.getMultiplexID()
};
srcPersistence.executeSQL(createMapping, Persistence.TEMP_CTX, multiplexArg);
}
/**
* A generator method which provides a suitable SQL to insert all records from an extent table
* to another based on an in-database primary key mapping. This will work only if the pk mapping
* was already created.
*
* @param srcExtent
* The name of the source extent temp-table.
* @param dstExtent
* The name of the destination extent temp-table.
* @param extColumnListSrc
* A string containing the columns from which the copy should be done.
* @param extColumnListDst
* A string containing the columns to which the copy should be done.
* @param pkMappingTableName
* The name of the table which stores the primary key mapping.
*
* @return The SQL which copies the records from one source extent table to a destination one.
*/
private String getExtentInsertSql(String srcExtent,
String dstExtent,
String extColumnListSrc,
String extColumnListDst,
String pkMappingTableName)
{
StringBuilder sql = new StringBuilder();
sql.append("insert into ").append(dstExtent)
.append(" (").append(extColumnListDst)
.append(") direct sorted select ").append(extColumnListSrc)
.append(" from ").append(srcExtent)
.append(" join ").append(pkMappingTableName)
.append(" on ").append(srcExtent)
.append(".").append(ReservedProperty.PARENT__ID.column)
.append(" = ").append(pkMappingTableName)
.append(".").append(SRC__PK)
.append(" order by ").append(srcExtent)
.append(".").append(ReservedProperty.PARENT__ID.column)
.append(", ").append(ReservedProperty.LIST__INDEX.column);
return sql.toString();
}
/**
* Special generator method which provides a suitable SQL to insert into a destination table
* only default values. This is based on an in-databse primary key mapping in order to correctly
* set the parent id. The list index will be locally generated and the values are set to default.
* This should be used when a source table exists, but an separate extent table for the source doesn't.
*
* @param extent
* The size of the extent for which the insert sql should be generated.
* @param dstExtent
* The name of the destination extent table.
* @param extColumnListSrc
* A list of fields from which the copy should be done. This should be formed only by ?,
* which will be set on the default values.
* @param extColumnListDst
* A list of the names of the extent field properties in the destination.
* @param pkMappingTableName
* The name of the in database pk mapping from which the parent-id will be extracted.
* @param uid
* Value used to generate unique names for intermediate tables.
*
* @return The SQL which creates new default records in the destination table.
*/
private String getSimpleExtentInsertSql(int extent,
String dstExtent,
String extColumnListSrc,
String extColumnListDst,
String pkMappingTableName,
long uid)
{
StringBuilder source = new StringBuilder();
source.append("(VALUES (0)");
for (int i = 1; i < extent; i++)
{
source.append(", (").append(i).append(")");
}
if (ALLOW_UID)
{
source.append(") as ").append(LIST__INDEX__TABLE).append("_").append(uid);
}
else
{
source.append(") as ").append(LIST__INDEX__TABLE);
}
source.append("(").append(ReservedProperty.LIST__INDEX.column).append(")");
StringBuilder sql = new StringBuilder();
sql.append("insert into ").append(dstExtent)
.append(" (").append(extColumnListDst)
.append(") direct sorted select ").append(extColumnListSrc)
.append(" from ").append(source.toString())
.append(" join ").append(pkMappingTableName)
.append(" order by ").append(pkMappingTableName)
.append(".").append(DST__PK)
.append(", ").append(ReservedProperty.LIST__INDEX.column);
return sql.toString();
}
/**
* A generator method which will create a database-level primary key mapping which can be used
* (through join) in order to speed up the copy process.
*
* @param srcTable
* The name of the source table.
* @param dstTable
* The name of the destination table.
* @param sqlIndexName
* The SQL name of the index used in the main table copy process
* @param extraWhereClauses
* A list of extra where clauses that need to be appended.
* @param sort
* The sort clause used in the main table copy process.
* @param pkMappingTableName
* The name which should be used for the generated table.
*
* @return The SQL which creates a database level primary key mapping.
*/
private String getPkMappingSql(String srcTable,
String dstTable,
String sqlIndexName,
List<String> extraWhereClauses,
String sort,
String pkMappingTableName)
{
StringBuilder sql = new StringBuilder();
sql.append("insert into ").append(pkMappingTableName).append(" direct sorted ")
.append("select next pk for (").append(dstTable).append(", ?)")
.append(" as \"").append(DST__PK).append("\",").append(Session.PK).append(" as \"").append(SRC__PK)
.append("\",").append(TempRecord._PEER_ROWID).append(" from ").append(srcTable).append(" ")
.append("use index (").append(sqlIndexName).append(") ")
.append("where ").append(MULTIPLEX_FIELD_NAME).append(" = ? ");
if (extraWhereClauses != null)
{
for (String clause : extraWhereClauses)
{
sql.append(" and ").append(clause);
}
}
sql.append(" order by ").append(sort);
return sql.toString();
}
/**
* A generator method which will copy the records from a source table into a destination table
* based on an already existing database-level pk mapping.
*
* @param srcTable
* The name of the source table.
* @param dstTable
* the name of the destination table.
* @param columnListSrc
* A string containing the columns from which the copy should be done.
* @param columnListDst
* A string containing the columns to which the copy should be done.
* @param sort
* The sort clause
* @param pkMappingTableName
* The table which contains the pk mapping.
*
* @return The SQL which copies records from the source to the destination based on the provided
* pk mapping.
*/
private String getShadowCopySql(String srcTable,
String dstTable,
String columnListSrc,
String columnListDst,
String sort,
String pkMappingTableName)
{
StringBuilder sql = new StringBuilder();
sql.append("insert into ").append(dstTable);
if (columnListDst.length() > 0)
{
sql.append(" ( ").append(columnListDst).append(") ");
}
else
{
sql.append(columnListDst);
}
sql.append(" direct sorted select ").append(columnListSrc)
.append(" from ").append(srcTable)
.append(" join ").append(pkMappingTableName)
.append(" on ").append(srcTable)
.append(".").append(Session.PK)
.append(" = ").append(pkMappingTableName)
.append(".").append(SRC__PK)
.append(" order by ").append(sort);
return sql.toString();
}
/**
* A generator method which will copy the records from a source table into a destination table.
*
* @param srcTable
* The name of the source table.
* @param dstTable
* the name of the destination table.
* @param columnListSrc
* A string containing the columns from which the copy should be done.
* @param columnListDst
* A string containing the columns to which the copy should be done.
* @param sqlIndexName
* The name of the index which should be used when copying records.
* @param sort
* The sort clause.
*
* @return The SQL which copies records from the source to the destination
*/
private String getFastTableCopySql(String srcTable,
String dstTable,
String columnListSrc,
String columnListDst,
String sqlIndexName,
String sort)
{
StringBuilder sql = new StringBuilder();
sql.append("insert into ").append(dstTable);
if (columnListDst.length() > 0)
{
sql.append(" ( ").append(columnListDst).append(") ");
}
else
{
sql.append(columnListDst);
}
sql.append(" direct sorted select ").append(columnListSrc)
.append(" from ").append(srcTable)
.append(" use index (").append(sqlIndexName)
.append(") where ").append(MULTIPLEX_FIELD_NAME)
.append(" = ? order by ").append(sort);
return sql.toString();
}
/**
* A generator method which will copy the records from a source table into a destination table
* with an additional condition on the recid of the source table.
*
* @param srcTable
* The name of the source table.
* @param dstTable
* the name of the destination table.
* @param columnListSrc
* A string containing the columns from which the copy should be done.
* @param columnListDst
* A string containing the columns to which the copy should be done.
* @param sqlIndexName
* The name of the index which should be used when copying records.
* @param extraWhereClauses
* Other where clauses to be included. Used to implement append-mode.
* @param sort
* The sort clause.
*
* @return The SQL which copies records from the source to the destination
*/
private String getSafeTableCopySqlNoMapping(String srcTable,
String dstTable,
String columnListSrc,
String columnListDst,
String sqlIndexName,
List<String> extraWhereClauses,
String sort)
{
StringBuilder sql = new StringBuilder();
sql.append("insert into ").append(dstTable);
if (columnListDst.length() > 0)
{
sql.append(" ( ").append(columnListDst).append(") ");
}
else
{
sql.append(columnListDst);
}
sql.append(" direct sorted select ").append(columnListSrc)
.append(" from ").append(srcTable)
.append(" use index (").append(sqlIndexName)
.append(") where ").append(MULTIPLEX_FIELD_NAME)
.append(" = ?");
// .append("and ").append(ReservedProperty.ID.column)
// .append(" < ?");
if (extraWhereClauses != null)
{
for (String whereClause : extraWhereClauses)
{
sql.append(" and ").append(whereClause);
}
}
sql.append(" order by ").append(sort);
return sql.toString();
}
/**
* This builds the columns from which the records should be copied. Note that the first columns
* are given through parameters: this should stand for the pk and _multiplex. However, when we will
* insert into the destination, the pk is assigned automatically by the database, while the _multiplex is
* fixed.
*
* @param dstTableName
* The name of the destination table. This is used to use the NEXT PK functionality.
*
* @return A string builder containing the columns which should be copied in the right order.
* This is a string builder as it is expected that this column list will be completed
* with other names.
*/
private StringBuilder getSrcTempColumns(String dstTableName)
{
return new StringBuilder("next pk for (").append(dstTableName).append(", ?)")
.append(",?,")
.append(ReservedProperty._ERRORFLAG.column)
.append(",")
.append(ReservedProperty._ORIGINROWID.column)
.append(",")
.append(ReservedProperty._DATASOURCEROWID.column)
.append(",")
.append(ReservedProperty._ERRORSTRING.column)
.append(",")
.append(ReservedProperty._PEERROWID.column)
.append(",")
.append(ReservedProperty._ROWSTATE.column);
}
/**
* This builds a wildcard for the given table instead of the columns. The first columns are given
* through parameters: this should stand for the pk (recid) and _multiplex. When inserting into
* the destination, the pk is assigned automatically by the database and the multiplex is
* a prepared value. The pk and _multiplex need to be excepted from the wildcard selection,
* this is done using EXCEPT.
*
* @param dstTableName
* The name of the destination table. This is used to use the NEXT PK functionality.
* @param sourcetableName
* The name of the source table.
*
* @return A string builder containing the columns which should be copied in the right order.
* This is a string builder as it is expected that this column list will be completed
* with other names.
*/
private StringBuilder getSrcTempWildcard(String dstTableName, String sourcetableName)
{
return new StringBuilder("next pk for (").append(dstTableName).append(", ?)")
.append(",?,")
.append(sourcetableName).append(".* EXCEPT (")
.append(sourcetableName).append(".")
.append(ReservedProperty.ID.column).append(",")
.append(sourcetableName).append(".")
.append(MULTIPLEX_FIELD_NAME).append(")");
}
/**
* A particular method which builds the columns from which the records should be copied in a context
* where an intermediate database-level pk mapping is used. For this case, the destination pk won't be
* generated automatically by the table using NEXT PK, but extracted from the provided pk mapping.
*
* @param pkMappingTableName
* The database-level pk mapping which is used when doing the copy.
*
* @return A string builder containing the columns which should be copied in the right order.
* This is a string builder as it is expected that this column list will be completed
* with other names.
*/
private StringBuilder getSrcMappedColumns(String pkMappingTableName)
{
return new StringBuilder(pkMappingTableName)
.append(".").append(DST__PK).append(",?,")
.append(ReservedProperty._ERRORFLAG.column).append(",")
.append(ReservedProperty._ORIGINROWID.column).append(",")
.append(ReservedProperty._DATASOURCEROWID.column).append(",")
.append(ReservedProperty._ERRORSTRING.column).append(",")
// we need to qualify _peerrowid, as this field is both in the pkMappingTable
// and source.
// The qualifier doesn't matter, they should be equal anyways
.append(pkMappingTableName).append(".")
.append(ReservedProperty._PEERROWID.column).append(",")
.append(ReservedProperty._ROWSTATE.column);
}
/**
* A particular method which builds a wildcard from which the records should be copied in a context
* where an intermediate database-level pk mapping is used. For this case, the destination pk won't be
* generated automatically by the table using NEXT PK, but extracted from the provided pk mapping.
* Note that pk and _multiplex need to be excepted from the wildcard selection of the source table,
* this is done using EXCEPT.
*
* @param pkMappingTableName
* The database-level pk mapping which is used when doing the copy.
* @param sourceTableName
* The name of the source table.
*
* @return A string builder containing a wildcard for the source table which should be copied
* in the right order.
*/
private StringBuilder getSrcMappedWildcard(String pkMappingTableName,
String sourceTableName)
{
return new StringBuilder().append(pkMappingTableName).append(".").append(DST__PK)
.append(",?,")
.append(sourceTableName).append(".* EXCEPT (")
.append(sourceTableName).append(".")
.append(ReservedProperty.ID.column).append(",")
.append(sourceTableName).append(".")
.append(MULTIPLEX_FIELD_NAME).append(")");
}
/**
* This builds the columns to which the records should be copied. Note that all columns are fixed.
*
* @return A string builder containing the target columns in the right order.
* This is a string builder as it is expected that this column list will be completed
* with other names.
*/
private StringBuilder getDstTempColumns()
{
return new StringBuilder(Session.PK).append(",")
.append(MULTIPLEX_FIELD_NAME).append(",")
.append(ReservedProperty._ERRORFLAG.column).append(",")
.append(ReservedProperty._ORIGINROWID.column).append(",")
.append(ReservedProperty._DATASOURCEROWID.column).append(",")
.append(ReservedProperty._ERRORSTRING.column).append(",")
.append(ReservedProperty._PEERROWID.column).append(",")
.append(ReservedProperty._ROWSTATE.column);
}
/**
* This extracts information from the source and destination tables. The most important
* part is the ordering of the properties - which can be done through a specified
* property mapping. Also, this will identify the extent fields (and the mapping of them
* between the source and destination).
*
* @return {@code true} if the processing didn't required loose-mode, so that the order of
* the properties in the source matches the one in the destination.
*/
private boolean processProperties()
{
srcSimpleProps = new ArrayList<>();
dstSimpleProps = new ArrayList<>();
extentFields = new HashMap<>();
if (!simpleCopy)
{
// maybe the order is preserved, so there is no need for complex management here
Map<Property, Property> propMapping = new HashMap<>();
for (Map.Entry<DatumAccess, DatumAccess> prop : propsMap.entrySet())
{
Property srcProp = srcMeta.getFieldInfo(prop.getKey().getPropertyName());
Property dstProp = dstMeta.getFieldInfo(prop.getValue().getPropertyName());
propMapping.put(dstProp, srcProp);
if (srcProp.extent == 0)
{
srcSimpleProps.add(srcProp);
dstSimpleProps.add(dstProp);
}
else
{
List<Property[]> extents =
extentFields.computeIfAbsent(srcProp.extent, k -> new ArrayList<>());
extents.add(new Property[] { srcProp, dstProp });
}
}
// we need to make sure we fill all the destination properties accordingly
// if they are not in the property mapping, we need to set them to a default value
Iterator<Property> iterSrc = srcMeta.getFields(false);
Iterator<Property> iterDst = dstMeta.getFields(false);
boolean orderPreserved = true;
while (iterDst.hasNext())
{
Property propDst = iterDst.next();
if (orderPreserved)
{
Property propSrc = iterSrc.hasNext() ? iterSrc.next() : null;
if (propSrc == null || propMapping.get(propDst) != propSrc)
{
orderPreserved = false;
}
}
if (propMapping.get(propDst) != null || getInitialValue(dstMeta, propDst) == null)
{
// this destination property was already resolved or
// it allows a null default, so no need to enforce special default value
continue;
}
if (propDst.extent == 0)
{
// mark the lack of a source property with null - this will be replaced with
// the default value when generating the SQLs
srcSimpleProps.add(null);
dstSimpleProps.add(propDst);
}
else
{
List<Property[]> extents =
extentFields.computeIfAbsent(propDst.extent, k -> new ArrayList<>());
extents.add(new Property[] { null, propDst });
}
}
orderPreserved = orderPreserved && !iterSrc.hasNext();
return orderPreserved;
}
else
{
Iterator<Property> iter = dstMeta.getFields(false);
while (iter.hasNext())
{
Property prop = iter.next();
if (prop.extent == 0)
{
srcSimpleProps.add(prop);
dstSimpleProps.add(prop);
}
else
{
List<Property[]> extents =
extentFields.computeIfAbsent(prop.extent, k -> new ArrayList<>());
extents.add(new Property[] { prop, prop });
}
}
return true;
}
}
/**
* A helper method which will append a source and destination property name to correct string builders:
* comma separated and having support for datetime-tz data type.
* Note that this should not be used for the first column (as it will have a leading comma).
* This also handles the cases when the source property is null, so a default value should be
* added to the source fields.
*
* @param srcMeta
* The dmo meta of the source.
* @param dstMeta
* The dmo meta of the destination. Used to retrieve eventual initial values.
* @param srcProp
* The source property which should be appended to the list of source fields.
* @param dstProp
* The destination property which should be appended to the list of source fields.
* @param srcSb
* The string builder which shall hold the information about the source fields.
* @param dstSb
* The string builder which shall hold the information about the destination fields.
* @param args
* The list of parameters which will be used in the SQL. Here the default will be appended
* if needed.
*/
private void appendColumnName(DmoMeta srcMeta,
DmoMeta dstMeta,
Property srcProp,
Property dstProp,
StringBuilder srcSb,
StringBuilder dstSb,
List<Object> args)
{
if (srcProp == null)
{
// this should be a default value
args.add(getInitialValue(dstMeta, dstProp));
srcSb.append(",").append("?");
dstSb.append(",").append(dstProp.column);
//TODO: add default for the extra DTZ__OFFSET field
}
else
{
srcSb.append(",").append(srcProp.column);
dstSb.append(",").append(dstProp.column);
if (srcProp._isDatetimeTz)
{
srcSb.append(",").append(srcProp.column).append(DDLGeneratorWorker.DTZ_OFFSET);
dstSb.append(",").append(dstProp.column).append(DDLGeneratorWorker.DTZ_OFFSET);
}
}
}
/**
* This will return the sort clause which should be used (in case am explicit index name is
* provided). If no index name is provided, the sort clause will be a simple combination between
* the multiplex and pk fields.
*
* @param indexName
* The index which should be taken in consideration when computing the sort clause.
* This can be null, resulting in a default sort clause.
*
* @return A sort clause suitable for the provided buffer and index name.
*/
private String getSortClause(String indexName)
throws PersistenceException
{
StringBuilder sort = null;
if (indexName != null)
{
Iterator<P2JIndex> iter = srcMeta.getDatabaseIndexes();
while (iter.hasNext())
{
P2JIndex idx = iter.next();
if (idx.getName().equals(indexName))
{
sort = new StringBuilder(MULTIPLEX_FIELD_NAME);
ArrayList<P2JIndexComponent> comps = idx.components();
for (int i = 0; i < comps.size(); i++)
{
P2JIndexComponent idxComp = comps.get(i);
String field = idxComp.getColumnName();
if (idxComp.isCharType() && srcDialect.needsComputedColumns())
{
field = srcDialect.getComputedColumnPrefix(!idxComp.isIgnoreCase()) + field;
}
sort.append(",");
srcDialect.orderByNulls(sort, field, null, true, false, idxComp.isMandatory());
}
if (!idx.isUnique())
{
sort.append(",").append(Session.PK);
}
break;
}
}
if (sort == null)
{
throw new PersistenceException("Could not find index for " + indexName);
}
}
if (sort == null)
{
sort = new StringBuilder(MULTIPLEX_FIELD_NAME).append(",").append(Session.PK);
}
return sort.toString();
}
/**
* This will retrieve the database-level pk mapping into a local mapping. This should be used only
* when the database-level pk mapping can't be used in the desired scenario.
*
* @param persistence
* A persistence instance which should be used in order to retrieve the database-level
* pk mapping.
* @param pkMappingTableName
* The name of the table which stored the database-level pk mapping.
*
* @return A map between the source pk and a group formed by the destination pk and source peer id
*/
private Map<Long, Long[]> getMapping(Persistence persistence, String pkMappingTableName)
throws PersistenceException
{
Map<Long, Long[]> srcToDstPks = new HashMap<>();
ScrollableResults<Long> pkMapping =
persistence.executeSQLQuery("select * from " + pkMappingTableName, Persistence.TEMP_CTX, null);
while (pkMapping.next())
{
Long dstPk = pkMapping.get(0, Long.class);
Long srcPk = pkMapping.get(1, Long.class);
Long srcPeerId = pkMapping.get(2, Long.class);
srcToDstPks.put(srcPk, new Long[] {dstPk, srcPeerId});
}
return srcToDstPks;
}
/**
* Retrieve the initial value of a certain property. The meta is used in order to retrieve
* the property metas which can be checked against the provided property and find the initial value.
*
* @param meta
* The meta of the dmo for the provided property.
* @param prop
* The property for which the initial value should be retrieved.
*
* @return The initial value of the provided property.
*/
private Object getInitialValue(DmoMeta meta, Property prop)
{
RecordMeta recordMeta = meta.getRecordMeta();
PropertyMeta[] propMetas = recordMeta.getPropertyMeta(false);
for (int i = 0; i < propMetas.length; i++)
{
PropertyMeta propMeta = propMetas[i];
if (propMeta.getId() == prop.id)
{
return propMeta.getInitialValue();
}
}
return null;
}
/**
* This method will call for {@link #copyTable} with table specific parameters. Also this
* will trigger {@link #processProperties} which will analyze the tables and compute
* a list of their properties (and extents) in a copy suitable order.
*
* @param fastCopyContext
* The context that has the information required.
* @param loose
* If the copy is in loose mode.
*/
private void copyTable(FastCopyContext fastCopyContext,
boolean loose)
throws PersistenceException
{
FastCopyBundleElement bundleElement;
// force a mapping if required or if in append mode
if (appendMode || this.forceMapping || !extentFields.isEmpty())
{
if (!this.forceMapping && extentFields.isEmpty())
{
bundleElement = safeTableCopyNoMapping(fastCopyContext,
srcSimpleProps,
dstSimpleProps,
getSortClause(),
loose);
}
else
{
bundleElement = safeTableCopy(fastCopyContext,
srcSimpleProps,
dstSimpleProps,
getSortClause(),
assembleUidMapping(pkMappingName),
loose);
generatesPkMapping = true;
}
}
else
{
bundleElement = copyTable(fastCopyContext,
srcSimpleProps,
dstSimpleProps,
getSortClause(),
loose);
}
fastCopyBundle.addBundleElement(bundleElement);
}
/**
* This method will call for {@link #getSortClause} with table specific parameters. This will
* also cache the sort clause.
*
* @return The sort clause which should be used in the table copy.
*/
private String getSortClause()
throws PersistenceException
{
if (sort == null)
{
sort = getSortClause(srcImplicitIndexName);
}
return sort;
}
/**
* Compute the destination table update SQL.
*
* @return String
* The {@code dstTableSql} field.
*/
private String getDstTableSql()
{
String dstTableSql = "update " + dstMeta.getSqlTableName() +
" set " + TempRecord._PEER_ROWID + " = ? " +
" where " + Session.PK + " = ?";
return dstTableSql;
}
/**
* This method will call for {@link #copyExtentFields} with table specific parameters. This will
* also try to create a database-level primary key mapping if it doesn't already exist.
*
* @param fastCopyContext
* The context that has the required information.
*/
private void copyExtentFields(FastCopyContext fastCopyContext)
throws PersistenceException
{
createPkMapping(fastCopyContext);
copyExtentFields(extentFields,
assembleUidMapping(pkMappingName),
uid);
}
/**
* This method will call for {@link #createPkMapping} with table specific parameters. This will
* take care that a single pk mapping will be online at once.
*
* @param fastCopyContext
* The context that has the required information.
*/
private void createPkMapping(FastCopyContext fastCopyContext)
throws PersistenceException
{
if (!generatesPkMapping)
{
createPkMapping(fastCopyContext,
getSortClause(),
assembleUidMapping(pkMappingName),
true);
generatesPkMapping = true;
}
}
/**
* This method will call for {@link #getMapping} with table specific parameters. This will
* cache eventually the mapping. Also, this may trigger a database-level pk mapping if it
* doesn't already exist.
*
* @param fastCopyContext
* The context that has the required information.
*
* @return Initializes the cache for the primary key after the copying of tables or
* returns the cache if it was already created.
*/
private Map<Long, Long[]> getMapping(FastCopyContext fastCopyContext)
throws PersistenceException
{
if (!fastCopyContext.hasMapping)
{
throw new IllegalStateException("Can't provide a PK mapping after the copy was done!");
}
return getMapping(srcPersistence, assembleUidMapping(pkMappingName));
}
/**
* Append the <code>uid</code> value to properties in order to generate
* unique names for the temporary and intermediary tables used in operations
* or storing indexes,
*
* @param property
* The property to which <code>uid</code> will be attached.
*
* @return A string representing a unique name.
*/
private String assembleUidMapping(String property)
{
if (!ALLOW_UID)
{
return property;
}
return property + "_" + uid;
}
/**
* Builder/Context for a {@code FastCopyHelper}.
*/
static class FastCopyContext
{
/** Is this context designated for copying before tables? */
boolean isBefore = false;
/** Flag identifying if the source and destination tables have the same structure. */
boolean simpleCopy = false;
/** Flag that indicates if the copy should happen with APPEND. */
boolean appendMode = false;
/** Flag that indicates if this is a loose copy process. */
boolean looseCopyMode = false;
/** Flag that indicates if the mapping is mandatory to be generated at the end. */
boolean forceMapping = false;
/** The source buffer. */
TemporaryBuffer srcBuf = null;
/** The destination buffer. */
TemporaryBuffer dstBuf = null;
/** The property mapping. */
Map<DatumAccess, DatumAccess> propsMap = null;
/** {@code true} if at the current state of the execution, the mapping is in the database */
boolean hasMapping = false;
/**
* Default constructor.
*/
public FastCopyContext()
{
}
/**
* Setter for the {@code isBefore} field.
*
* @param isBefore
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setIsBefore(boolean isBefore)
{
this.isBefore = isBefore;
return this;
}
/**
* Setter for the {@code simpleCopy} field.
*
* @param simpleCopy
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setSimpleCopy(boolean simpleCopy)
{
this.simpleCopy = simpleCopy;
return this;
}
/**
* Setter for the {@code appendMode} field.
*
* @param appendMode
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setAppendMode(boolean appendMode)
{
this.appendMode = appendMode;
return this;
}
/**
* Setter for the {@code looseCopyMode} field.
*
* @param looseCopyMode
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setLooseCopyMode(boolean looseCopyMode)
{
this.looseCopyMode = looseCopyMode;
return this;
}
/**
* Setter for the {@code srcBuf} field.
*
* @param srcBuf
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setSourceBuffer(TemporaryBuffer srcBuf)
{
this.srcBuf = srcBuf;
return this;
}
/**
* Setter for the {@code dstBuf} field.
*
* @param dstBuf
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setDestinationBuffer(TemporaryBuffer dstBuf)
{
this.dstBuf = dstBuf;
return this;
}
/**
* Setter for the {@code propsMap} field.
*
* @param propsMap
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setPropsMap(Map<DatumAccess, DatumAccess> propsMap)
{
this.propsMap = propsMap;
return this;
}
/**
* Force the intermediate pk-mapping when copying. This is used when there is a
* normalized extent table copy or before table copy, so the mapping of the
* source-destination pk is needed.
*
* @param forceMapping
* The new value.
*
* @return FastCopyContext
* This instance.
*/
public FastCopyContext setForceMapping(boolean forceMapping)
{
this.forceMapping = forceMapping;
return this;
}
/**
* Function to build a {@code FastCopyHelper} based on the information stored in this context.
*
* @return FastCopyHelper
* The created instance based on this context.
*/
public FastCopyHelper build()
{
return new FastCopyHelper(this);
}
}
/**
* Class that acts as the key for the {@code fastCopyCache}.
*/
static class FastCopyKey
{
/** The class of the source buffer. */
final Class<? extends Record> sourceBufferClass;
/** The class of the destination buffer. */
final Class<? extends Record> destinationBufferClass;
/** A bitmask that indicates which copying mode is being used. */
final byte copyMode;
/**
* Basic constructor.
*
* @param fastCopyContext
* The context with the required information.
*/
FastCopyKey(FastCopyContext fastCopyContext)
{
sourceBufferClass = fastCopyContext.srcBuf != null ?
fastCopyContext.srcBuf.getDMOImplementationClass() : null;
destinationBufferClass = fastCopyContext.dstBuf != null ?
fastCopyContext.dstBuf.getDMOImplementationClass() : null;
copyMode = (byte) ((fastCopyContext.appendMode ? 1 << 0 : 0) +
(fastCopyContext.looseCopyMode ? 1 << 1 : 0) +
(fastCopyContext.forceMapping ? 1 << 2 : 0));
}
/**
* Function that computes the hash.
*/
@Override
public int hashCode()
{
int hash = 17;
hash = 37 * hash + (sourceBufferClass != null ? sourceBufferClass.hashCode() : 0);
hash = 37 * hash + (destinationBufferClass != null ? destinationBufferClass.hashCode() : 0);
hash = 37 * hash + copyMode;
return hash;
}
/**
* Function that checks the equality between two {@code FastCopyKey}.
*/
@Override
public boolean equals(Object o)
{
if (!(o instanceof FastCopyKey))
{
return false;
}
FastCopyKey that = (FastCopyKey) o;
return this.sourceBufferClass == that.sourceBufferClass &&
this.destinationBufferClass == that.destinationBufferClass &&
this.copyMode == that.copyMode;
}
}
/**
* Class that stores a bundle of SQL that will need to be executed in order to perform the copy process.
*/
static class FastCopyBundle
{
/** List of elements used in the copy process. */
final List<FastCopyBundleElement> bundle;
/**
* Default constructor.
*/
FastCopyBundle()
{
bundle = new ArrayList<>();
}
/**
* Check if the bundle has already been computed. If so, one can start the execution process directly.
*
* @return boolean
* {@code true} if the bundle has already been computed, {@code false} otherwise.
*/
boolean isBundleComputed()
{
return bundle.size() > 0;
}
/**
* Add an element to the bundle.
*
* @param element
* The element that will be added to the bundle.
*/
void addBundleElement(FastCopyBundleElement element)
{
bundle.add(element);
}
/**
* Method that executes all the elements in the bundle for the given context.
* In case an error occurs, the session will rollback to a savepoint defined before the
* execution began.
*
* @param context
* The context with the required information.
*/
void executeSqls(FastCopyContext context)
throws PersistenceException
{
Session session = context.srcBuf.getSession();
Savepoint sp = session.setSavepoint();
for (int i = 0; i < bundle.size(); i++)
{
FastCopyBundleElement element = bundle.get(i);
try
{
element.execute(context);
}
catch (PersistenceException exc)
{
session.rollbackSavepoint(sp);
throw exc;
}
}
}
}
/**
* Class that denotes an element of the bundle.
* This can be extended depending on the type of the element.
*/
class FastCopyBundleElement
{
/** The SQL that will be executed. */
final String sql;
/** A {@code Function} that will generate the arguments for the SQL, depending on the given context. */
final Function<FastCopyContext, Object[]> generateArgs;
/**
* Default constructor with two parameters.
*
* @param sql
* The SQL that will be used.
* @param generateArgs
* The {@code Function} used to generate the arguments.
*/
FastCopyBundleElement(String sql, Function<FastCopyContext, Object[]> generateArgs)
{
this.sql = sql;
this.generateArgs = generateArgs;
}
/**
* Method that executes the SQL of this element, with the computed arguments.
*
* @param context
* The context used to generate arguments.
*/
void execute(FastCopyContext context)
throws PersistenceException
{
srcPersistence.executeSQL(sql, Persistence.TEMP_CTX, generateArgs.apply(context));
}
}
}