LockTableUpdater.java

/*
** Module   : LockTableUpdater.java
** Abstract : Manages metadata lock records for a particular, primary database
**
** Copyright (c) 2013-2025, Golden Code Development Corporation.
**
** -#- -I- --Date-- ---------------------------------------Description---------------------------------------
** 001 ECF 20131101 Created initial version.
** 002 ECF 20140116 Use session ID as user number (id) for lock events.
** 003 ECF 20140213 Change query strings to use JPA-style query substitution placeholders.
** 004 ECF 20140509 Fix HQL_FILENUM query string to use new computed column prefix.
** 005 ECF 20140509 Encoded more maintainable version of previous fix 004 at OM's suggestion.
** 006 ECF 20140918 Only roll back in process event if in a transaction.
** 007 ECF 20151203 Fixed HQL_FILENUM query, which was searching on historical table name, while
**                  the substitution parameter provided by the record lock event is the converted
**                  table name.
** 008 ECF 20160228 Name the update worker thread.
** 009 ECF 20160421 Replaced expensive, sequence-based primary key request with simple counter.
** 010 ECF 20160514 Replaced expensive HQL queries with more efficient SQL queries.
** 011 ECF 20160609 SQL performance improvement.
** 012 ECF 20160822 Fixed failing lock metadata updates.
** 013 OM  20190111 Added support for LockChain field.
** 014 ECF 20200906 New ORM implementation.
** 015 CA  20200924 Replaced Method.invoke with ReflectASM.
**     OM  20201012 Force use locally cached meta information instead of map lookup.
**     IAS 20220324 Performance optimization in the UserTableStatUpdater style.
**     ECF 20220630 Fixed MinimalLock getter/setter names to match conversion of historical names in _Lock
**                  table. Code cleanup.
**     TJD 20220504 Upgrade do Java 11 minor changes
** 016 GBB 20230512 Logging methods replaced by CentralLogger/ConversionStatus.
** 017 GBB 20230825 SecurityManager session methods calls updated.
** 018 HC  20240222 Enabled JMX on FWD Client.
** 019 TJD 20240208 Java 17 dependencies updates
** 020 RAA 20240423 Modified constructor to also include the database configuration.
** 021 OM  20241128 Multi-tenant runtime support: selected the proper persistence context, eventually
**                  based on [sharedDb] parameter.
** 022 OM  20250131 A bit of code maintenance.
*/

/*
** This program is free software: you can redistribute it and/or modify
** it under the terms of the GNU Affero General Public License as
** published by the Free Software Foundation, either version 3 of the
** License, or (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU Affero General Public License for more details.
**
** You may find a copy of the GNU Affero GPL version 3 at the following
** location: https://www.gnu.org/licenses/agpl-3.0.en.html
** 
** Additional terms under GNU Affero GPL version 3 section 7:
** 
**   Under Section 7 of the GNU Affero GPL version 3, the following additional
**   terms apply to the works covered under the License.  These additional terms
**   are non-permissive additional terms allowed under Section 7 of the GNU
**   Affero GPL version 3 and may not be removed by you.
** 
**   0. Attribution Requirement.
** 
**     You must preserve all legal notices or author attributions in the covered
**     work or Appropriate Legal Notices displayed by works containing the covered
**     work.  You may not remove from the covered work any author or developer
**     credit already included within the covered work.
** 
**   1. No License To Use Trademarks.
** 
**     This license does not grant any license or rights to use the trademarks
**     Golden Code, FWD, any Golden Code or FWD logo, or any other trademarks
**     of Golden Code Development Corporation. You are not authorized to use the
**     name Golden Code, FWD, or the names of any author or contributor, for
**     publicity purposes without written authorization.
** 
**   2. No Misrepresentation of Affiliation.
** 
**     You may not represent yourself as Golden Code Development Corporation or FWD.
** 
**     You may not represent yourself for publicity purposes as associated with
**     Golden Code Development Corporation, FWD, or any author or contributor to
**     the covered work, without written authorization.
** 
**   3. No Misrepresentation of Source or Origin.
** 
**     You may not represent the covered work as solely your work.  All modified
**     versions of the covered work must be marked in a reasonable way to make it
**     clear that the modified work is not originating from Golden Code Development
**     Corporation or FWD.  All modified versions must contain the notices of
**     attribution required in this license.
*/

package com.goldencode.p2j.persist.meta;

import java.lang.reflect.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.concurrent.locks.*;
import java.util.logging.*;
import com.goldencode.p2j.jmx.*;
import com.goldencode.p2j.persist.*;
import com.goldencode.p2j.persist.Record;
import com.goldencode.p2j.persist.lock.*;
import com.goldencode.p2j.persist.orm.*;
import com.goldencode.p2j.security.*;
import com.goldencode.p2j.security.SecurityManager;
import com.goldencode.p2j.util.*;
import com.goldencode.p2j.util.logging.*;
import com.goldencode.proxy.*;

/**
 * A {@link com.goldencode.p2j.persist.lock.LockListener lock listener} implementation which
 * receives record lock events and updates the metadata lock in-memory map accordingly. 
 * When application executes query against the _Lock table the information about active locks is 
 * persisted into the meta database before executing the query. 
 * Persisted locks are deleted from the meta table in a separate thread when they are released. 
 */
public final class LockTableUpdater
implements LockListener
{
   /** Logger */
   private static final CentralLogger log = CentralLogger.get(LockTableUpdater.class.getName());
   
   /** Worker which processes record lock events in a dedicated server thread */
   private final UpdateWorker worker;
   
   /** Numeric ID of the user in the current session */
   private final ContextLocal<integer> sessionID = new ContextLocal<integer>()
   {
      /** Security manager */
      private final SecurityManager securityManager;
      
      /** List of reclaimed user numbers */
      private final List<integer> reclaimedSessionIDs;
      
      /** Highest allocated session ID to date (may no longer be active) */
      private int highSessionID = 0;
      
      // Constructor for this anonymous inner class
      {
         securityManager = SecurityManager.getInstance();
         reclaimedSessionIDs = (securityManager == null ? new LinkedList<>() : null);
      }
      
      @Override
      protected integer initialValue()
      {
         integer sessionID;
         
         if (securityManager != null)
         {
            sessionID = new integer(securityManager.sessionSm.getSessionId());
         }
         else
         {
            // fallback method, though we should always have a security manager in server mode
            synchronized (this)
            {
               if (!reclaimedSessionIDs.isEmpty())
               {
                  sessionID = reclaimedSessionIDs.remove(0);
               }
               else
               {
                  sessionID = new integer(++highSessionID);
               }
            }
         }
         
         return sessionID;
      }
      
      protected void cleanup(integer value)
      {
         if (securityManager == null)
         {
            synchronized (this)
            {
               reclaimedSessionIDs.add(value);
            }
         }
      }
   };
   
   /**
    * Constructor.
    * @param   primary 
    *          Primary database
    * @param   metaDatabase
    *          Metadata database.
    * @param   cfg
    *          The database configuration.
    */
   public LockTableUpdater(Database primary, Database metaDatabase, DatabaseConfig cfg)
   {
      this.worker = new UpdateWorker(PersistenceFactory.getInstance(metaDatabase, cfg), primary);
   }
   
   /**
    * Persist locks in the meta database
    */
   public void persist()
   {
      worker.persist();
   }
   /**
    * Method which is called by a {@link LockManager} in response to a lock event. Does some
    * preliminary processing and sanity checking, then queues the event for further processing on
    * a dedicated thread.
    * 
    * @param   event
    *          Object which describes the lock event.
    */
   public void lockChanged(RecordLockEvent event)
   {
      LockType oldType = event.getOldType();
      LockType newType = event.getNewType();
      
      // if old and new lock types are both NONE, nothing to do, but log a warning, because this
      // suggests a problem
      // TODO: does this work with NO-WAIT flag, too?
      if (LockType.NONE.equals(oldType) && LockType.NONE.equals(newType))
      {
         if (log.isLoggable(Level.WARNING))
         {
            log.log(Level.WARNING, "Unexpected lock event:  " + event);
            
            return;
         }
      }
      
      event.setUserid(sessionID.get());
      worker.addEvent(event);
   }
   
   /**
    * A worker containing a queue to which record lock events are written from multiple user
    * threads/contexts, and from which those events are read in a dedicated, daemon thread. The
    * idea is to offload the potentially more time-consuming work of querying and updating the
    * metadata database from a dedicated thread, to minimize any performance penalty of this work
    * on user sessions. Since this naturally introduces the possibility of the lock metadata to
    * get out of sync with the actual lock states in the lock manager, a synchronization mechanism
    * is used to ensure any pending updates are flushed to the database before any query against
    * this table is executed. This way, only those user sessions actually querying the lock
    * metadata are affected by any lag in updating the metadata from record lock events.
    * <p>
    * Note that some analysis of the lock event has to occur in the user session in which the
    * lock event occurred, but as much of the work as possible is deferred until the separate
    * thread processes the events.
    */
   private static class UpdateWorker
   implements LockTableUpdaterStatMBean
   {
      /** Metadata lock table name */
      private static final String LOCK_TABLE = "meta_lock";
      
      /** SQL query to find a lock record by PK */
      private static final String SQL_LOCK =
         "select m." + Session.PK + " from " + LOCK_TABLE + " m where m." + Session.PK + " = ?1";
      
      /** The offset between a table and its surrogate chain id. */
      private static final int CHAIN_OFFSET = 1_000_000;
      
      /** MetaLock DMO implementation class */
      private static final Class<? extends Record> dmoClass;
      
      /** Map of <code>MinimalLock</code> methods to <code>MetaLockImpl</code> methods */
      private static final Map<Method, Method> methodMap = new HashMap<>();
      
      /*
       * Before this static block is executed the MetadataManager.initialize() is processed. At
       * that time, if the _Lock meta table is present configured in configuration file, the
       * specialized DMO interface - dmoName is already registered with DmoMetadataManager.
       */
      static
      {
         Class<? extends Record> implementingClass;
         String dmoName = DmoMetadataManager.getDmoBasePackage() + "._meta.MetaLock";
         try
         {
            DmoMeta dmoInfo = DmoMetadataManager.getDmoInfo(Class.forName(dmoName));
            implementingClass = dmoInfo.getImplementationClass();
            
            // map MinimalLock methods to corresponding dmoClass methods; if we try to use
            // MinimalLock methods with a MetaLockImpl object, we will raise
            // IllegalArgumentException during Method.invoke(), because our object will not be an
            // instance of MinimalLock
            for (Method ifcMeth : MinimalLock.class.getMethods())
            {
               Method dmoMeth = implementingClass.getMethod(ifcMeth.getName(), ifcMeth.getParameterTypes());
               methodMap.put(ifcMeth, dmoMeth);
            }
         }
         catch (ClassNotFoundException | IllegalArgumentException exc)
         {
            // not a fatal error; application does not use _Lock table
            implementingClass = null;
         }
         catch (NoSuchMethodException exc)
         {
            // our MinimalLock does not match the full _Lock DMO from conversion time
            throw new RuntimeException("Error preparing MetaLock implementation class", exc);
         }
         dmoClass = implementingClass;
      }
      
      /** Number of processed lock events. */
      private static final AtomicLong lockEvents = new AtomicLong(0);
      
      /** Number of records in VST. */
      private static final AtomicLong persistedLocks = new AtomicLong(0);
      
      /** Temporary holder of the number of records in VST - before commit/rollback. */
      private static final AtomicLong persistedLocks_ = new AtomicLong(0);
      
      /** Number of VST records deleted not in batch. */
      private static final AtomicLong deletesFromVST = new AtomicLong(0);
      
      /** Number of 'persist' calls */
      private static final AtomicLong persistCalls = new AtomicLong(0);
      
      /** Executor for operations with VST */
      private final Executor executor = Executors.newSingleThreadExecutor(
         new ThreadFactory()
         {
            @Override
            public Thread newThread(Runnable r)
            {
               Thread workThread = new AssociatedThread(r);
               workThread.setDaemon(true);
               workThread.setName("Lock metadata update thread");
               return workThread;
            }
         }
      );
      
      /** Locks' map */
      private final ConcurrentMap<LockRecId, MinimalLockImpl> locks = new ConcurrentSkipListMap<>();
      
      /** Map version */
      private final AtomicLong mapVersion = new AtomicLong();
      
      /** Meta table vesrion */
      private final AtomicLong databaseVersion = new AtomicLong(0);
      
      /** Locks' map guard */
      private final ReentrantReadWriteLock guard = new ReentrantReadWriteLock();
      
      /** database name */
      private final Database primary;
      
      /** Persistence helper object for metadata database */
      private final Persistence persistence;
      
      /** Proxy object for the current metadata lock DMO */
      private final MinimalLock proxy;
      
      /** Invocation handler which maps proxy method calls to underlying DMO methods */
      private final Handler handler = new Handler();
      
      /** Next available primary key for a new lock metadata record */
      private final AtomicLong nextPrimaryKey = new AtomicLong(1L);
      
      /** Dummy empty class to be used as the super class for proxy. */
      public static class Empty {}
      
      /**
       * Constructor.
       * 
       * @param   persistence
       *          Persistence helper object for metadata database.
       */
      UpdateWorker(Persistence persistence, Database primary)
      {
         this.persistence = persistence;
         this.primary = primary;
         this.proxy = ProxyFactory.getProxy(Empty.class, new Class<?>[] { MinimalLock.class }, handler);
         FwdServerJMX.register(new LockTableUpdaterStat(this), primary.getName() + ".locks");
         executor.execute(() -> {}); // start executor
      }
      
      /**
       * Submit persisting locks task to the meta database and wait for end
       */
      public void persist()
      {
         CountDownLatch latch = new CountDownLatch(1);
         executor.execute(() -> {
            try 
            {
               doPersist();
               latch.countDown();
            }
            catch(Throwable t)
            {
               log.log(Level.WARNING, "Failed to persist lock table data", t);
            }
         });
         try
         {
            latch.await();
         }
         catch (InterruptedException ignore)
         {
         }
      }
      
      /**
       * Persist locks to the meta database
       */
      public void doPersist()
      {
         persistCalls.incrementAndGet();
         persistedLocks_.set(persistedLocks.get());
         guard.writeLock().lock();
         boolean commit = false;
         try
         {
            if (databaseVersion.get() >= mapVersion.get())
            {
               return;
            }
            commit = persistence.beginTransaction(Persistence.META_CTX);
            for(Iterator<Map.Entry<LockRecId, MinimalLockImpl>> it = locks.entrySet().iterator();
                it.hasNext(); )
            {
               Map.Entry<LockRecId, MinimalLockImpl> e = it.next();
               e.getValue().persist();
               if (e.getValue().deleted)
               {
                  it.remove();
               }
            }
            if (commit)
            {
               commit = false;   // to avoid double-rollback in catch block if commit fails
               persistence.commit(Persistence.META_CTX);
            }
            for (Iterator<Map.Entry<LockRecId, MinimalLockImpl>> it = locks.entrySet().iterator();
                 it.hasNext(); )
            {
               Map.Entry<LockRecId, MinimalLockImpl> e = it.next();
               e.getValue().commit();
               if (e.getValue().deleted)
               {
                  it.remove();
               }
            }
            persistedLocks.set(persistedLocks_.get());
            databaseVersion.set(mapVersion.get());
         }
         catch (PersistenceException e1)
         {
            log.log(Level.WARNING, "Failed to persist lock data", e1);
            locks.values().forEach(MinimalLockImpl::rollback);
            if (commit)
            {
               try
               {
                  persistence.rollback(Persistence.META_CTX);
               }
               catch (PersistenceException ignore)
               {
               }
            }
         }
         finally
         {
            guard.writeLock().unlock();
         }
      }
      
      /**
       * Add a record lock event to the queue, locking out other write and read operations until
       * the queue is updated. Notify a the worker thread when the new event has been added.
       * 
       * @param   event
       *          Record lock event to be added.
       */
      void addEvent(RecordLockEvent event)
      {
         lockEvents.incrementAndGet();
         guard.readLock().lock();
         try
         {
            mapVersion.incrementAndGet();
            LockRecId lrId = new LockRecId(event);
            if (event.getNewType() == LockType.NONE )
            {
               MinimalLockImpl ml = locks.get(lrId);
               if (ml != null && ml.persisted.get() >= 0) 
               {
                  // lock was already persisted, VST cleanup is required
                  ml.guardedUpdate(event);
                  executor.execute(() -> {
                     try
                     {
                        if (ml.persist())
                        {
                           locks.remove(lrId);
                           deletesFromVST.incrementAndGet();
                           persistedLocks.decrementAndGet();
                        }
                     }
                     catch(Throwable t)
                     {
                       log.log(Level.WARNING, "Cannot delete lock data for " + ml.eventString, t); 
                     }
                  });
               }
               else
               {
                  locks.remove(lrId);
               }
               return;
            }
            MinimalLockImpl ml = locks.computeIfAbsent(lrId, k -> new MinimalLockImpl(event));
            if (!ml.version.compareAndSet(-1, 0)) // not a new lock, should be updated
            {
               ml.guardedUpdate(event);
            }
         }
         finally
         {
            guard.readLock().unlock();
         }
      }
      
      /**
       * Extract an integral lock record ID from the lock event. This will be a positive value
       * for records and a negative value for tables.
       * 
       * @param   event
       *          A record lock event.
       * 
       * @return  See above.
       */
      public long getLockRecordID(RecordLockEvent event) 
      {
         Long recordID = event.getIdentifier().getRecordID();
         long recID = (recordID != null) ? (Long) recordID : -getFileNum(event);
         
         return recID;
      }
      
      /**
       * Get the file number associated with a lock event.
       * 
       * @param   event
       *          Record lock event.
       * 
       * @return  File number for the table associated with the event.
       */
      private Integer getFileNum(RecordLockEvent event)
      {
         RecordIdentifier<String> ident = event.getIdentifier();
         String table = ident.getTable();
         Integer fileNum;
         if (MetadataManager.inUse(MetadataManager._FILE))
         {
            // converted table name is stored in MetaFile.userMisc
            fileNum = MetadataManager.getFileNum(primary, table);
            
            if (fileNum == null)
            {
               String msg = "Table '" + table + "' not found in metadata [" + event + "]";
               throw new IllegalStateException(msg);
            }
         }
         else
         {
            // the _File metadata is not available
            fileNum = 0;
         }
         return fileNum;
      }
      
      /**
       * Invocation handler which delegates calls to the {@link MinimalLock} proxy methods to the
       * backing DMO instance.
       */
      private static class Handler
      implements InvocationHandler
      {
         /** DMO delegate whose methods ultimately are invoked */
         private Object delegate = null;
         
         /**
          * Process method invocations on the {@link MinimalLock} proxy by delegating them to the
          * backing DMO instance.
          * 
          * @param   proxy
          *          Proxy on which the enclosing class invokes methods.
          * @param   method
          *          Method which was invoked.
          * @param   args
          *          Method arguments.
          */
         @Override
         public Object invoke(Object proxy, Method method, Object[] args)
         throws Throwable
         {
            Method dmoMethod = methodMap.get(method);
            
            return Utils.invoke(dmoMethod, delegate, args);
         }
         
         /**
          * Set the backing DMO delegate instance.
          * 
          * @param   delegate
          *          DMO whose methods will be invoked.
          */
         void setDelegate(Object delegate)
         {
            this.delegate = delegate;
         }
      }
      
      /** In-memory holder for the _Lock VST data*/ 
      private class MinimalLockImpl
      {
         /** Lock guard */
         public final ReentrantReadWriteLock guard = new ReentrantReadWriteLock();
         
         /** Lock version */
         public final AtomicLong version = new AtomicLong(-1);
         
         /** Lock version last persisted */
         public final AtomicLong persisted = new AtomicLong(-1);
         
         /** Flag indicating that the corresponding record was already deleted in VST */
         public volatile boolean deleted = false;
         
         /** Temporary holder for the 'deleted' flag -used until commit/rollback */
         public volatile boolean deleted_ = false;
         
         /** Primary key */
         public final long key;
         
         /** Lock Id */
         public final int64 lockId;
         
         /** User */
         public final integer lockUsr;
         
         /** Lock name */
         public final character lockName;
         
         /** Table fileNum */ 
         public final integer lockTable;
         
         /** Record Id */
         public final int64 lockRecId;
         
         /** Chain */
         public final integer lockChain;
         
         /** Old lock type */
         private LockType oldType;
         
         /** New lock type */
         private LockType newType;
         
         /** Lock type string */
         private character lockType;
         
         /** Lock flags */
         private character lockFlags;
         
         /** Event string representation (for logging) */
         private String eventString;
         
         /**
          * Constructor
          * 
          * @param event
          *        Lock event
          */
         public MinimalLockImpl(RecordLockEvent event)
         {
            key = nextPrimaryKey.incrementAndGet();
            
            // set user name and user id
            String subject = event.getOsUserName();
            if (subject == null)
            {
               subject = event.getLocker().getSubject();
            }
            lockName = new character(subject);
            lockUsr = event.getUserid();
            
            // get file number for table
            Integer fileNum = getFileNum(event);
            
            // assign other properties that will not change with new events
            lockId = new int64(key);    // re-use primary key
            lockTable = new integer(fileNum);
            lockRecId = new int64(getLockRecordID(event));
            
            // In ProgressOE, the locking table is divided into chains anchored in a hash table.
            // These chains are provided for fast lookup of record locks by Rec-id.
            // FWD OTOH, relies on underlying database for recid/rowid lookup so the chain is at
            // least irrelevant. In order to have a substitute and keep things simple, we will
            // assume that each table has a single chain and its value is computed as an fixed
            // offset from the table ID.
            lockChain = new integer(fileNum + CHAIN_OFFSET);
            
            update(event);
         }
         
         /**
          * Guarded update holder with event data
          * 
          * @param   event 
          *          lock event
          */
         public void guardedUpdate(RecordLockEvent event)
         {
            guard.writeLock().lock();
            try
            {
               update(event);
               version.incrementAndGet();
            }
            finally
            {
               guard.writeLock().unlock();
            }
         }
         
         /** 
          * Update holder with event data
          * 
          * @param   event 
          *          lock event
          */
         public void update(RecordLockEvent event)
         {
            this.oldType = event.getOldType();
            this.newType = event.getNewType();
            String code = newType.isExclusive() ? "X" : "S"; // TODO:  get this right
            if (oldType.isShare())
            {
               code += " U"; // an upgrade
            }
            // From ABL online manual: values of flags field specify:
            //    S: a share lock,
            //    X: an exclusive lock,
            //    U: a lock upgraded from share to exclusive,
            //    L: a lock in limbo,
            //    Q: a queued lock,
            //    K: a lock kept across transaction end boundary,
            //    J: a lock is part of a JTA transaction,
            //    C: a lock is in create mode for JTA,
            //    E: a lock wait timeout has expired on this queued lock.
            this.lockFlags = new character(code);
            
            boolean isTable = (event.getIdentifier() instanceof TableIdentifier);
            this.lockType = new character(isTable ? "TAB" : "REC");
            this.eventString = String.format("%s: %d[%d, %d]", event,
                                             key, lockRecId.longValue(), lockUsr.intValue());
         }
         
         /**
          * Create record in the _Lock VST
          */
         private Record createRecord() throws PersistenceException
         {
            Record dmo = null;
            
            try
            {
               // create new DMO instance and back the proxy with it
               dmo = dmoClass.getDeclaredConstructor().newInstance();
               dmo.initialize(null, true);
               handler.setDelegate(dmo);
               
               // assign primary key
               dmo.primaryKey(key);
               // lock meta record for writing
               RecordIdentifier<String> metaIdent = new RecordIdentifier<>(dmoClass.getName(),
                                                                           dmo.primaryKey());
               persistence.lock(LockType.EXCLUSIVE, metaIdent, true, Persistence.META_CTX);
               proxy.setLockName(lockName);
               proxy.setLockUsr(lockUsr);
               proxy.setLockId(lockId); // re-use primary key
               proxy.setLockTable(lockTable);
               proxy.setLockRecId(lockRecId);
               proxy.setLockChain(lockChain);
            }
            catch (ReflectiveOperationException exc)
            {
               String msg = String.format("Error creating meta lock record for event %s", eventString);
               
               throw new PersistenceException(msg, exc);
            }
            return dmo;
         }
         
         /** Find record in the _Lock VST */
         private Record findRecord() throws PersistenceException
         {
            Long id = persistence.getSingleSQLResult(SQL_LOCK, Persistence.META_CTX, new Object[] { key });
            Record dmo = null;
            
            if (id != null)
            {
               RecordIdentifier<String> metaIdent = new RecordIdentifier<>(dmoClass.getName(), id);
               persistence.lock(LockType.EXCLUSIVE, metaIdent, true, Persistence.META_CTX);
               dmo = persistence.quickLoad(metaIdent, Persistence.META_CTX);
            }
            
            if (dmo == null)
            {
               throw new PersistenceException("No lock record found for lock event:  " + eventString);
            }
            
            handler.setDelegate(dmo);
            
            return dmo;
         }
         
         /**
          * Guarded persist lock data in the _Lock VST.
          * 
          * @return  <code>true</code> if success
          * @throws  PersistenceException 
          *          on database error.
          */
         public boolean persist()
         throws PersistenceException
         {
            guard.writeLock().lock();
            try
            {
               return doPersist();
            }
            finally
            {
               guard.writeLock().unlock();
            }
         }
         
         /**
          * Commit the changes of the lock data state
          */
         private void commit()
         {
            persisted.set(version.get());
            deleted = deleted_;
            deleted_ = false;
         }
         
         /**
          * Roll back the changes of the lock data state
          */
         private void rollback()
         {
            deleted_ = false;
         }
         
         /**
          * Persist lock data in the _Lock VST.
          * 
          * @return  {@code true} on success.
          *
          * @throws  PersistenceException
          *          On database error.
          */
         private boolean doPersist()
         throws PersistenceException
         {
            if (persisted.get() >= version.get() || deleted)
            {
               return false;
            }
            Record dmo = null;
            boolean commit = false;
            
            try
            {
               commit = persistence.beginTransaction(Persistence.META_CTX);
               
               if (persisted.get() < 0)
               {
                  dmo = createRecord();
                  if (log.isLoggable(Level.FINE))
                  {
                     log.fine("created lock record [" + eventString + "]");
                  }
                  persistedLocks_.incrementAndGet();
               }
               else
               {
                  try
                  {
                     dmo = findRecord();
                  }
                  catch (Exception e)
                  {
                     log.log(Level.WARNING, "No record found", e);
                     try
                     {
                        if (commit)
                        {
                           this.rollback();
                           persistence.rollback(Persistence.META_CTX);
                        }
                     }
                     catch (PersistenceException ignore)
                     {
                     }
                     return false;
                  }
                  if (LockType.NONE.equals(newType))
                  {
                     persistence.delete(dmo);
                     persistedLocks_.decrementAndGet();
                     deleted_ = true;
                     if (log.isLoggable(Level.FINE))
                     {
                        log.fine("deleted lock record [" + eventString + "]");
                     }
                  }
                  else
                  {
                     proxy.setLockFlags(lockFlags);
                     proxy.setLockType(lockType);
                     if (log.isLoggable(Level.FINE))
                     {
                        log.fine("updated lock record [" + eventString + "]");
                     }
                  }
               }
               persistence.save(dmo, dmo.primaryKey());
               persistence.flush(Persistence.META_CTX);
               
               if (commit)
               {
                  commit = false;   // to avoid double-rollback in catch block if commit fails
                  persistence.commit(Persistence.META_CTX);
                  this.commit();
               }
               return true;
            }
            catch (PersistenceException exc)
            {
               if (log.isLoggable(Level.SEVERE))
               {
                  String msg = String.format("Error updating _lock table for %s (%s)",
                                             persistence.getDatabase(Persistence.META_CTX).toString(),
                                             eventString);
                  log.log(Level.SEVERE, msg, exc);
               }
               if (commit)
               {
                  try
                  {
                     this.rollback();
                     persistence.rollback(Persistence.META_CTX);
                  }
                  catch (PersistenceException ignore)
                  {
                  }
               }
               else
               {
                  // propagate exception to the outer block.
                  throw exc;
               }
            }
            finally
            {
               if (dmo != null)
               {
                  RecordIdentifier<String> metaIdent = new RecordIdentifier<>(dmoClass.getName(),
                                                                              dmo.primaryKey());
                  try
                  {
                     persistence.lock(LockType.NONE, metaIdent, true, Persistence.META_CTX);
                  }
                  catch (LockUnavailableException exc)
                  {
                     // won't be thrown from lock release
                  }
               }
            }
            return false;
         }
      }
      
      /**
       * Lock record id
       */
      private class LockRecId
      implements Comparable<LockRecId>
      {
         /** lock Record Id */
         public final long lockRec;
         
         /** lock User Id */
         public final int lockUsr;
         
         /**
          * Constructor
          * 
          * @param   event
          *          Lock event
          */
         public LockRecId(RecordLockEvent event) 
         {
            this.lockRec = getLockRecordID(event);
            this.lockUsr = event.getUserid().intValue();
         }
         
         /**
          * Compare with the given record id
          * 
          * @return  -1 if other record id is greater, 1 if other record is less, 0 if ids are equal
          */
         @Override
         public int compareTo(LockRecId o)
         {
            return this.lockRec < o.lockRec ? -1 : 
                   this.lockRec > o.lockRec ?  1 : 
                      this.lockUsr < o.lockUsr ? -1 :
                      this.lockUsr > o.lockUsr ?  1 : 0;
         }
      }
      
      /**
       * Get number of processed lock events.
       * 
       * @return  number of processed lock events.
       */
      @Override
      public long getLockEvents()
      {
         return lockEvents.get();
      }
      
      /**
       * Get number of active locks
       * 
       * @return  number of active locks.
       */
      @Override
      public long getActiveLocks()
      {
         return locks.size();
      }
      
      /**
       * Get number of locks in VST.
       * 
       * @return  number of locks in VST.
       */
      @Override
      public long getPersistedLocks()
      {
         return persistedLocks.get();
      }
      
      /**
       * Get number of VST records deleted not in batch.
       * 
       * @return  number of VST records deleted not in batch.
       */
      @Override
      public long getDeletesFromVST()
      {
         return deletesFromVST.get();
      }
      
      /**
       * Get number of 'persist' calls.
       * 
       * @return  number of 'persist' calls.
       */
      @Override
      public long getPersistCalls()
      {
         return persistCalls.get();
      }
      
      /**
       * Force 'persist' call
       */
      @Override
      public void flush()
      {
         persist();
      }
   }
   
   /**
    * Interface which defines the minimally required methods of the <code>MetaLock</code>
    * interface. This interface defines the proxy used to create and manipulate metadata lock
    * DMO instances. The DMO's methods cannot be invoked directly from code here, since the
    * <code>MetaLock</code> interface resides in an application-specific package, which is not
    * known at compile time.
    */
   public interface MinimalLock
   {
      /**
       * Getter: lockId
       * 
       * @return  lockId
       */
      public int64 getLockId();
      
      /**
       * Setter: lockId
       * 
       * @param   lockId
       *          lockId
       */
      public void setLockId(NumberType lockId);
      
      /**
       * Getter: Usr
       * 
       * @return  Usr
       */
      public integer getLockUsr();
      
      /**
       * Setter: Usr
       * 
       * @param   lockUsr
       *          Usr
       */
      public void setLockUsr(NumberType lockUsr);
      
      /**
       * Getter: Name
       * 
       * @return  Name
       */
      public character getLockName();
      
      /**
       * Setter: Name
       * 
       * @param   lockName
       *          Name
       */
      public void setLockName(Text lockName);
      
      /**
       * Getter: Type
       * 
       * @return  Type
       */
      public character getLockType();
      
      /**
       * Setter: Type
       * 
       * @param   lockType
       *          Type
       */
      public void setLockType(Text lockType);
      
      /**
       * Getter: Table
       * 
       * @return  Table
       */
      public integer getLockTable();
      
      /**
       * Setter: Table
       * 
       * @param   lockTable
       *          Table
       */
      public void setLockTable(NumberType lockTable);
      
      /**
       * Getter: RECID
       * 
       * @return  RECID
       */
      public int64 getLockRecId();
      
      /**
       * Setter: RECID
       * 
       * @param   lockRecId
       *          RECID
       */
      public void setLockRecId(NumberType lockRecId);
      
      /**
       * Getter: Chain
       *
       * @return  Chain
       */
      public integer getLockChain();
      
      /**
       * Setter: Chain
       *
       * @param   lockChain
       *          Chain
       */
      public void setLockChain(NumberType lockChain);
      
      /**
       * Getter: Flags
       * 
       * @return  Flags
       */
      public character getLockFlags();
      
      /**
       * Setter: Flags
       * 
       * @param   lockFlags
       *          Flags
       */
      public void setLockFlags(Text lockFlags);
   }
}