# IndexMetadataCollector.java

    /*
    ** Module   : IndexMetadataCollector.java
    ** Abstract : Collects index metadata and caches results for fast access.
    **
    ** Copyright (c) 2013-2020, Golden Code Development Corporation.
    **
    ** -#- -I- --Date-- --------------------------------------Description----------------------------------------
    ** 001 ECF 20200517 Refactored from Persistence class and reworked JDBC connection access.
    ** 002 OM  20200924 P2JIndexComponent carries multiple information to avoid map lookups for them.
    */

    /*
    ** This program is free software: you can redistribute it and/or modify
    ** it under the terms of the GNU Affero General Public License as
    ** published by the Free Software Foundation, either version 3 of the
    ** License, or (at your option) any later version.
    **
    ** This program is distributed in the hope that it will be useful,
    ** but WITHOUT ANY WARRANTY; without even the implied warranty of
    ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ** GNU Affero General Public License for more details.
    **
    ** You may find a copy of the GNU Affero GPL version 3 at the following
    ** location: https://www.gnu.org/licenses/agpl-3.0.en.html
    **
    ** Additional terms under GNU Affero GPL version 3 section 7:
    **
    **   Under Section 7 of the GNU Affero GPL version 3, the following additional
    **   terms apply to the works covered under the License.  These additional terms
    **   are non-permissive additional terms allowed under Section 7 of the GNU
    **   Affero GPL version 3 and may not be removed by you.
    **
    **   0. Attribution Requirement.
    **
    **     You must preserve all legal notices or author attributions in the covered
    **     work or Appropriate Legal Notices displayed by works containing the covered
    **     work.  You may not remove from the covered work any author or developer
    **     credit already included within the covered work.
    **
    **   1. No License To Use Trademarks.
    **
    **     This license does not grant any license or rights to use the trademarks
    **     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
    **     of Golden Code Development Corporation. You are not authorized to use the
    **     name Golden Code, FWD, or the names of any author or contributor, for
    **     publicity purposes without written authorization.
    **
    **   2. No Misrepresentation of Affiliation.
    **
    **     You may not represent yourself as Golden Code Development Corporation or FWD.
    **
    **     You may not represent yourself for publicity purposes as associated with
    **     Golden Code Development Corporation, FWD, or any author or contributor to
    **     the covered work, without written authorization.
    **
    **   3. No Misrepresentation of Source or Origin.
    **
    **     You may not represent the covered work as solely your work.  All modified
    **     versions of the covered work must be marked in a reasonable way to make it
    **     clear that the modified work is not originating from Golden Code Development
    **     Corporation or FWD.  All modified versions must contain the notices of
    **     attribution required in this license.
    */

    package com.goldencode.p2j.persist;

    import java.sql.*;
    import java.util.*;
    import java.util.concurrent.*;
    import com.goldencode.p2j.persist.dialect.*;
    import com.goldencode.p2j.persist.orm.*;

    /**
     * This class queries index metadata using JDBC in order to generate lists of {@link P2JIndex} objects,
     * which are made available by table name via the {@link #queryIndexData(Session, String)} method. The
     * first time a databases is registered with the persistence layer it has its schema scanned and index
     * metadata is cached for faster lookup on subsequent occasions.
     */
    public final class IndexMetadataCollector
    {
       /** A transient mapping of JDBC connections to column data type maps */
       private static final Map<Connection, Map<ColumnKey, Integer>> columnTypes = new WeakHashMap<>(4);

       /** Mappings of table names to index lists, organized by schema */
       private static Map<String, Map<String, List<P2JIndex>>> indexesByTable = new HashMap<>();

       /**
        * Query all index metadata from the given database, using a thread pool with 1 thread per
        * available CPU. Store this information for faster access later.
        *
        * @param   database
        *          Database from which to query index metadata.
        * @param   schema
        *          Schema containing index information.
        *
        * @throws  PersistenceException
        *          if there is an error accessing the database or querying metadata.
        */
       static void queryAllIndexData(Database database, String schema)
       throws PersistenceException
       {
          Map<String, List<P2JIndex>> map = indexesByTable.get(schema);
          if (map != null)
          {
             return;
          }

          map = new ConcurrentHashMap<>();

          Map<String, List<P2JIndex>> innerMap = map;

          // a "connection pool" of sessions to use for the thread pool work below
          int threads = Runtime.getRuntime().availableProcessors();
          final BlockingDeque<Session> sessionDeque = new LinkedBlockingDeque<>(threads);

          try
          {
             Session session = new Session(database);
             Connection conn = session.getConnection();
             DatabaseMetaData meta = conn.getMetaData();

             ExecutorService exec = Executors.newFixedThreadPool(threads);
             List<Future<PersistenceException>> futures = new LinkedList<>();

             // TODO: make dialect-aware schema and table type arguments
             ResultSet rs = meta.getTables(null, "public", "%", new String[] { "TABLE" });

             // populate the deque with database sessions
             sessionDeque.push(session);
             for (int i = 1; i < threads; i++)
             {
                sessionDeque.push(new Session(database));
             }

             while (rs.next())
             {
                String table = rs.getString("TABLE_NAME");

                Callable<PersistenceException> task = () ->
                {
                   try
                   {
                      Session s = sessionDeque.pop();
                      List<P2JIndex> list = queryIndexData(s, table);
                      sessionDeque.push(s);
                      innerMap.put(table, list);
                   }
                   catch (PersistenceException exc)
                   {
                      return exc;
                   }

                   return null;
                };

                futures.add(exec.submit(task));
             }

             for (Future<PersistenceException> future : futures)
             {
                try
                {
                   PersistenceException exc = future.get();
                   if (exc != null)
                   {
                      throw exc;
                   }
                }
                catch (ExecutionException | InterruptedException exc)
                {
                   throw new PersistenceException(exc);
                }
             }

             exec.shutdown();

             indexesByTable.put(schema, map);
          }
          catch (SQLException exc)
          {
             DBUtils.handleException(database, exc);

             throw new UnsupportedOperationException("Error querying table metadata for " + database, exc);
          }
          finally
          {
             if (DatabaseManager.isInitializing())
             {
                columnTypes.clear();
             }

             for (Session session : sessionDeque)
             {
                session.close();
             }
          }
       }

       /**
        * Gather a list of {@code P2JIndex} objects which describe the indices on the given table.  This method
        * uses the database metadata, if any, exposed by the database's JDBC driver.
        *
        * @param   session
        *          The session to be used. It provides both the database which contains {@code table} and the
        *          connection to it.
        * @param   table
        *          Name of table to inspect.
        *
        * @return  A list of {@code P2JIndex} objects, one per index found, or {@code null} if the JDBC metadata
        *          query throws {@code SQLException}, indicating a database error, or that the JDBC driver does
        *          not support the requested feature. An empty list is returned if the driver supported the query,
        *          but no indices were found.
        *
        * @throws  PersistenceException
        *          if there is an error getting a session or determining an indexed column's data type.
        * @throws  UnsupportedOperationException
        *          if the JDBC driver cannot query database metadata.
        */
       public static List<P2JIndex> queryIndexData(Session session, String table)
       throws PersistenceException
       {
          Database database = session.getDatabase();
          String schema = DatabaseManager.getSchema(database);
          Map<String, List<P2JIndex>> map = indexesByTable.get(schema);
          if (map != null)
          {
             List<P2JIndex> list = map.get(table);
             if (list != null)
             {
                return list;
             }
          }

          Dialect dialect = DatabaseManager.getDialect(database);
          String _table = dialect.prepareMetadataParameter(table);

          try
          {
             Connection conn = session.getConnection();
             DatabaseMetaData meta = conn.getMetaData();

             // the parameters will be prepared using dialect specific semantics
             ResultSet rs = meta.getIndexInfo(null, null, _table, false, false);

             Map<String, P2JIndex> indices = retrieveIndexData(rs, database, conn, table);

             return new ArrayList<>(indices.values());
          }
          catch (SQLException exc)
          {
             DBUtils.handleException(database, exc);

             throw new UnsupportedOperationException("Error querying index metadata for " + database, exc);
          }
       }

       /**
        * Given a the result set of an index metadata query (for a certain table),
        * this method is responsible for creating a map which will hold, for each
        * index, a {@link P2JIndex} object.
        *
        * @param   rs
        *          The {@link ResultSet} instance used to retrieve the index metadata.
        * @param   database
        *          Database which contains <code>table</code>.
        * @param   conn
        *          Database connection.
        * @param   table
        *          The table for which the indexes are retrieved.
        *
        * @return  A map with {@link P2JIndex} instances for each existing table index.
        *
        * @throws  PersistenceException
        *          if any database error is encountered.
        */
       static Map<String, P2JIndex> retrieveIndexData(ResultSet rs,
                                                      Database database,
                                                      Connection conn,
                                                      String table)
       throws PersistenceException
       {
          Dialect dialect = DatabaseManager.getDialect(database);
          Map<String, P2JIndex> indices = new LinkedHashMap<>();

          try
          {
             P2JIndex index = null;

             // all results must be processed using the dialect
             while (rs.next())
             {
                if (rs.getShort("TYPE") == DatabaseMetaData.tableIndexStatistic)
                {
                   continue;
                }

                String indexName = rs.getString("INDEX_NAME");
                indexName = dialect.processMetadataResult(indexName);
                if (indexName == null)
                {
                   continue;
                }

                index = indices.get(indexName);
                if (index == null)
                {
                   boolean nonUniq = rs.getBoolean("NON_UNIQUE");
                   index = new P2JIndex(table, indexName, !nonUniq, false, false);
                   indices.put(indexName, index);
                }

                String colName = rs.getString("COLUMN_NAME");
                colName = dialect.processMetadataResult(colName);

                // Determine case insensitivity in a dialect-specific way.
                boolean ignoreCase = dialect.isCaseInsensitiveColumn(colName);

                // Extract root column name in a dialect-specific way.
                colName = dialect.extractColumnName(colName);

                // Determine the data type of the indexed column.
                int dataType = getColumnDataType(database, conn, table, colName);
                boolean isChar = (dataType == Types.VARCHAR);

                String sort = rs.getString("ASC_OR_DESC");
                sort = dialect.processMetadataResult(sort);

                boolean descending = dialect.isMetadataSortDesc(sort);

                index.addComponent(null, null, colName, descending, isChar, isChar && ignoreCase, false);
             }

             if (index != null)
             {
                index.compact();
             }
          }
          catch (SQLException exc)
          {
             DBUtils.handleException(database, exc);

             throw new PersistenceException(exc);
          }

          return indices;
       }

       /**
        * Use database metadata to determine the data type of a database column.
        *
        * @param   database
        *          Target database.
        * @param   conn
        *          Database connection.
        * @param   table
        *          Target table.
        * @param   column
        *          Target column.
        *
        * @return  Data type of the given column, as one of the <code>java.sql.Types</code> constants.
        *
        * @throws  PersistenceException
        *          if there is any database error, or if the metadata query fails
        *          to produce a result for the given column.
        */
       private static int getColumnDataType(Database database, Connection conn, String table, String column)
       throws PersistenceException
       {
          Dialect dialect = DatabaseManager.getDialect(database);

          // the parameters will be prepared using dialect specific semantics
          String _public = dialect.prepareMetadataParameter("public");
          String _table  = dialect.prepareMetadataParameter(table);
          String _column = dialect.prepareMetadataParameter(column);

          try
          {
             DatabaseMetaData meta = null;

             if (DatabaseManager.isInitializing())
             {
                // Initialization optimization:
                // Try to gather up and cache all public schema column data types
                // in a single metadata query, rather than with many individual
                // queries.  The cache only lasts as long as the connection, which
                // is assumed to be shared across many of these data type
                // requests.
                Map<ColumnKey, Integer> types = columnTypes.computeIfAbsent(conn, c -> new HashMap<>());

                ColumnKey key = new ColumnKey(table, column);
                Integer dataType = types.get(key);

                if (dataType == null)
                {
                   meta = conn.getMetaData();
                   // TODO:  is "public" schema portable?
                   ResultSet rs = meta.getColumns(null, _public, null, null);

                   while (rs.next())
                   {
                      // all results must be processed using the dialect
                      String nextTab = rs.getString("TABLE_NAME");
                      nextTab = dialect.processMetadataResult(nextTab);

                      String nextCol = rs.getString("COLUMN_NAME");
                      nextCol = dialect.processMetadataResult(nextCol);

                      int nextType = rs.getInt("DATA_TYPE");
                      ColumnKey nextKey = new ColumnKey(nextTab, nextCol);
                      if (key.equals(nextKey))
                      {
                         dataType = nextType;
                      }
                      types.put(nextKey, nextType);
                   }
                }

                if (dataType != null)
                {
                   return dataType;
                }
             }

             // Either we are not in the persistence initialization phase or the
             // all-in-one strategy failed.  The latter could happen if a JDBC
             // driver does not honor an empty string for the column name pattern
             // in the meta.getColumns(...) call above.  Try an individual
             // metadata query for the exact table and column name instead.  This
             // is less efficient, but may be necessary with some drivers.
             // TODO:  is "public" schema portable?
             if (meta == null)
             {
                meta = conn.getMetaData();
             }

             // the parameters will be prepared using dialect specific semantics
             ResultSet rs = meta.getColumns(null, _public, _table, _column);
             if (rs.next())
             {
                return rs.getInt("DATA_TYPE");
             }

             throw new PersistenceException(
                "Unable to determine data type for column '" +
                table +
                ":" +
                column +
                "'");
          }
          catch (SQLException exc)
          {
             DBUtils.handleException(database, exc);

             throw new PersistenceException(exc);
          }
       }

       /**
        * A hash key for a unique combination of table and column name.
        */
       private static class ColumnKey
       {
          /** Table name */
          private final String table;

          /** Column name */
          private final String column;

          /**
           * Constructor.
           *
           * @param   table
           *          Table name.
           * @param   column
           *          Column name.
           */
          ColumnKey(String table, String column)
          {
             this.table = table;
             this.column = column;
          }

          /**
           * Test for equivalence with another object.
           *
           * @param   o
           *          Another <code>ColumnKey</code> instance.
           *
           * @return  <code>true</code> if <code>o</code> has same table and
           *          column name as this object;  else <code>false</code>.
           */
          @Override
          public boolean equals(Object o)
          {
             if (!(o instanceof ColumnKey))
             {
                return false;
             }
             ColumnKey that = (ColumnKey) o;

             return this.table.equals(that.table) && this.column.equals(that.column);
          }

          /**
           * A hash code implementation consisent with our overridden {@link
           * #equals(Object)} method.
           *
           * @return  Hash code.
           */
          @Override
          public int hashCode()
          {
             int result = 17;

             result = 37 * result + table.hashCode();
             result = 37 * result + column.hashCode();

             return result;
          }

          /**
           * Get string representation of this object.
           *
           * @return  String representation.
           */
          @Override
          public String toString()
          {
             return table + ":" + column;
          }
       }
    }