IndexMetadataCollector.java
/*
** Module : IndexMetadataCollector.java
** Abstract : Collects index metadata and caches results for fast access.
**
** Copyright (c) 2013-2020, Golden Code Development Corporation.
**
** -#- -I- --Date-- --------------------------------------Description----------------------------------------
** 001 ECF 20200517 Refactored from Persistence class and reworked JDBC connection access.
** 002 OM 20200924 P2JIndexComponent carries multiple information to avoid map lookups for them.
*/
/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
**
** Additional terms under GNU Affero GPL version 3 section 7:
**
** Under Section 7 of the GNU Affero GPL version 3, the following additional
** terms apply to the works covered under the License. These additional terms
** are non-permissive additional terms allowed under Section 7 of the GNU
** Affero GPL version 3 and may not be removed by you.
**
** 0. Attribution Requirement.
**
** You must preserve all legal notices or author attributions in the covered
** work or Appropriate Legal Notices displayed by works containing the covered
** work. You may not remove from the covered work any author or developer
** credit already included within the covered work.
**
** 1. No License To Use Trademarks.
**
** This license does not grant any license or rights to use the trademarks
** Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
** of Golden Code Development Corporation. You are not authorized to use the
** name Golden Code, FWD, or the names of any author or contributor, for
** publicity purposes without written authorization.
**
** 2. No Misrepresentation of Affiliation.
**
** You may not represent yourself as Golden Code Development Corporation or FWD.
**
** You may not represent yourself for publicity purposes as associated with
** Golden Code Development Corporation, FWD, or any author or contributor to
** the covered work, without written authorization.
**
** 3. No Misrepresentation of Source or Origin.
**
** You may not represent the covered work as solely your work. All modified
** versions of the covered work must be marked in a reasonable way to make it
** clear that the modified work is not originating from Golden Code Development
** Corporation or FWD. All modified versions must contain the notices of
** attribution required in this license.
*/
package com.goldencode.p2j.persist;
import java.sql.*;
import java.util.*;
import java.util.concurrent.*;
import com.goldencode.p2j.persist.dialect.*;
import com.goldencode.p2j.persist.orm.*;
/**
* This class queries index metadata using JDBC in order to generate lists of {@link P2JIndex} objects,
* which are made available by table name via the {@link #queryIndexData(Session, String)} method. The
* first time a databases is registered with the persistence layer it has its schema scanned and index
* metadata is cached for faster lookup on subsequent occasions.
*/
public final class IndexMetadataCollector
{
/** A transient mapping of JDBC connections to column data type maps */
private static final Map<Connection, Map<ColumnKey, Integer>> columnTypes = new WeakHashMap<>(4);
/** Mappings of table names to index lists, organized by schema */
private static Map<String, Map<String, List<P2JIndex>>> indexesByTable = new HashMap<>();
/**
* Query all index metadata from the given database, using a thread pool with 1 thread per
* available CPU. Store this information for faster access later.
*
* @param database
* Database from which to query index metadata.
* @param schema
* Schema containing index information.
*
* @throws PersistenceException
* if there is an error accessing the database or querying metadata.
*/
static void queryAllIndexData(Database database, String schema)
throws PersistenceException
{
Map<String, List<P2JIndex>> map = indexesByTable.get(schema);
if (map != null)
{
return;
}
map = new ConcurrentHashMap<>();
Map<String, List<P2JIndex>> innerMap = map;
// a "connection pool" of sessions to use for the thread pool work below
int threads = Runtime.getRuntime().availableProcessors();
final BlockingDeque<Session> sessionDeque = new LinkedBlockingDeque<>(threads);
try
{
Session session = new Session(database);
Connection conn = session.getConnection();
DatabaseMetaData meta = conn.getMetaData();
ExecutorService exec = Executors.newFixedThreadPool(threads);
List<Future<PersistenceException>> futures = new LinkedList<>();
// TODO: make dialect-aware schema and table type arguments
ResultSet rs = meta.getTables(null, "public", "%", new String[] { "TABLE" });
// populate the deque with database sessions
sessionDeque.push(session);
for (int i = 1; i < threads; i++)
{
sessionDeque.push(new Session(database));
}
while (rs.next())
{
String table = rs.getString("TABLE_NAME");
Callable<PersistenceException> task = () ->
{
try
{
Session s = sessionDeque.pop();
List<P2JIndex> list = queryIndexData(s, table);
sessionDeque.push(s);
innerMap.put(table, list);
}
catch (PersistenceException exc)
{
return exc;
}
return null;
};
futures.add(exec.submit(task));
}
for (Future<PersistenceException> future : futures)
{
try
{
PersistenceException exc = future.get();
if (exc != null)
{
throw exc;
}
}
catch (ExecutionException | InterruptedException exc)
{
throw new PersistenceException(exc);
}
}
exec.shutdown();
indexesByTable.put(schema, map);
}
catch (SQLException exc)
{
DBUtils.handleException(database, exc);
throw new UnsupportedOperationException("Error querying table metadata for " + database, exc);
}
finally
{
if (DatabaseManager.isInitializing())
{
columnTypes.clear();
}
for (Session session : sessionDeque)
{
session.close();
}
}
}
/**
* Gather a list of {@code P2JIndex} objects which describe the indices on the given table. This method
* uses the database metadata, if any, exposed by the database's JDBC driver.
*
* @param session
* The session to be used. It provides both the database which contains {@code table} and the
* connection to it.
* @param table
* Name of table to inspect.
*
* @return A list of {@code P2JIndex} objects, one per index found, or {@code null} if the JDBC metadata
* query throws {@code SQLException}, indicating a database error, or that the JDBC driver does
* not support the requested feature. An empty list is returned if the driver supported the query,
* but no indices were found.
*
* @throws PersistenceException
* if there is an error getting a session or determining an indexed column's data type.
* @throws UnsupportedOperationException
* if the JDBC driver cannot query database metadata.
*/
public static List<P2JIndex> queryIndexData(Session session, String table)
throws PersistenceException
{
Database database = session.getDatabase();
String schema = DatabaseManager.getSchema(database);
Map<String, List<P2JIndex>> map = indexesByTable.get(schema);
if (map != null)
{
List<P2JIndex> list = map.get(table);
if (list != null)
{
return list;
}
}
Dialect dialect = DatabaseManager.getDialect(database);
String _table = dialect.prepareMetadataParameter(table);
try
{
Connection conn = session.getConnection();
DatabaseMetaData meta = conn.getMetaData();
// the parameters will be prepared using dialect specific semantics
ResultSet rs = meta.getIndexInfo(null, null, _table, false, false);
Map<String, P2JIndex> indices = retrieveIndexData(rs, database, conn, table);
return new ArrayList<>(indices.values());
}
catch (SQLException exc)
{
DBUtils.handleException(database, exc);
throw new UnsupportedOperationException("Error querying index metadata for " + database, exc);
}
}
/**
* Given a the result set of an index metadata query (for a certain table),
* this method is responsible for creating a map which will hold, for each
* index, a {@link P2JIndex} object.
*
* @param rs
* The {@link ResultSet} instance used to retrieve the index metadata.
* @param database
* Database which contains <code>table</code>.
* @param conn
* Database connection.
* @param table
* The table for which the indexes are retrieved.
*
* @return A map with {@link P2JIndex} instances for each existing table index.
*
* @throws PersistenceException
* if any database error is encountered.
*/
static Map<String, P2JIndex> retrieveIndexData(ResultSet rs,
Database database,
Connection conn,
String table)
throws PersistenceException
{
Dialect dialect = DatabaseManager.getDialect(database);
Map<String, P2JIndex> indices = new LinkedHashMap<>();
try
{
P2JIndex index = null;
// all results must be processed using the dialect
while (rs.next())
{
if (rs.getShort("TYPE") == DatabaseMetaData.tableIndexStatistic)
{
continue;
}
String indexName = rs.getString("INDEX_NAME");
indexName = dialect.processMetadataResult(indexName);
if (indexName == null)
{
continue;
}
index = indices.get(indexName);
if (index == null)
{
boolean nonUniq = rs.getBoolean("NON_UNIQUE");
index = new P2JIndex(table, indexName, !nonUniq, false, false);
indices.put(indexName, index);
}
String colName = rs.getString("COLUMN_NAME");
colName = dialect.processMetadataResult(colName);
// Determine case insensitivity in a dialect-specific way.
boolean ignoreCase = dialect.isCaseInsensitiveColumn(colName);
// Extract root column name in a dialect-specific way.
colName = dialect.extractColumnName(colName);
// Determine the data type of the indexed column.
int dataType = getColumnDataType(database, conn, table, colName);
boolean isChar = (dataType == Types.VARCHAR);
String sort = rs.getString("ASC_OR_DESC");
sort = dialect.processMetadataResult(sort);
boolean descending = dialect.isMetadataSortDesc(sort);
index.addComponent(null, null, colName, descending, isChar, isChar && ignoreCase, false);
}
if (index != null)
{
index.compact();
}
}
catch (SQLException exc)
{
DBUtils.handleException(database, exc);
throw new PersistenceException(exc);
}
return indices;
}
/**
* Use database metadata to determine the data type of a database column.
*
* @param database
* Target database.
* @param conn
* Database connection.
* @param table
* Target table.
* @param column
* Target column.
*
* @return Data type of the given column, as one of the <code>java.sql.Types</code> constants.
*
* @throws PersistenceException
* if there is any database error, or if the metadata query fails
* to produce a result for the given column.
*/
private static int getColumnDataType(Database database, Connection conn, String table, String column)
throws PersistenceException
{
Dialect dialect = DatabaseManager.getDialect(database);
// the parameters will be prepared using dialect specific semantics
String _public = dialect.prepareMetadataParameter("public");
String _table = dialect.prepareMetadataParameter(table);
String _column = dialect.prepareMetadataParameter(column);
try
{
DatabaseMetaData meta = null;
if (DatabaseManager.isInitializing())
{
// Initialization optimization:
// Try to gather up and cache all public schema column data types
// in a single metadata query, rather than with many individual
// queries. The cache only lasts as long as the connection, which
// is assumed to be shared across many of these data type
// requests.
Map<ColumnKey, Integer> types = columnTypes.computeIfAbsent(conn, c -> new HashMap<>());
ColumnKey key = new ColumnKey(table, column);
Integer dataType = types.get(key);
if (dataType == null)
{
meta = conn.getMetaData();
// TODO: is "public" schema portable?
ResultSet rs = meta.getColumns(null, _public, null, null);
while (rs.next())
{
// all results must be processed using the dialect
String nextTab = rs.getString("TABLE_NAME");
nextTab = dialect.processMetadataResult(nextTab);
String nextCol = rs.getString("COLUMN_NAME");
nextCol = dialect.processMetadataResult(nextCol);
int nextType = rs.getInt("DATA_TYPE");
ColumnKey nextKey = new ColumnKey(nextTab, nextCol);
if (key.equals(nextKey))
{
dataType = nextType;
}
types.put(nextKey, nextType);
}
}
if (dataType != null)
{
return dataType;
}
}
// Either we are not in the persistence initialization phase or the
// all-in-one strategy failed. The latter could happen if a JDBC
// driver does not honor an empty string for the column name pattern
// in the meta.getColumns(...) call above. Try an individual
// metadata query for the exact table and column name instead. This
// is less efficient, but may be necessary with some drivers.
// TODO: is "public" schema portable?
if (meta == null)
{
meta = conn.getMetaData();
}
// the parameters will be prepared using dialect specific semantics
ResultSet rs = meta.getColumns(null, _public, _table, _column);
if (rs.next())
{
return rs.getInt("DATA_TYPE");
}
throw new PersistenceException(
"Unable to determine data type for column '" +
table +
":" +
column +
"'");
}
catch (SQLException exc)
{
DBUtils.handleException(database, exc);
throw new PersistenceException(exc);
}
}
/**
* A hash key for a unique combination of table and column name.
*/
private static class ColumnKey
{
/** Table name */
private final String table;
/** Column name */
private final String column;
/**
* Constructor.
*
* @param table
* Table name.
* @param column
* Column name.
*/
ColumnKey(String table, String column)
{
this.table = table;
this.column = column;
}
/**
* Test for equivalence with another object.
*
* @param o
* Another <code>ColumnKey</code> instance.
*
* @return <code>true</code> if <code>o</code> has same table and
* column name as this object; else <code>false</code>.
*/
@Override
public boolean equals(Object o)
{
if (!(o instanceof ColumnKey))
{
return false;
}
ColumnKey that = (ColumnKey) o;
return this.table.equals(that.table) && this.column.equals(that.column);
}
/**
* A hash code implementation consisent with our overridden {@link
* #equals(Object)} method.
*
* @return Hash code.
*/
@Override
public int hashCode()
{
int result = 17;
result = 37 * result + table.hashCode();
result = 37 * result + column.hashCode();
return result;
}
/**
* Get string representation of this object.
*
* @return String representation.
*/
@Override
public String toString()
{
return table + ":" + column;
}
}
}